2024-11-17 22:39:43,245 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 22:39:43,261 main DEBUG Took 0.013899 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 22:39:43,262 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 22:39:43,262 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 22:39:43,263 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 22:39:43,264 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,276 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 22:39:43,293 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,294 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,295 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,297 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,298 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,299 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,300 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,301 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,302 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,303 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,303 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,304 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,304 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,305 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,305 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,306 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,306 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,307 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,307 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,308 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,308 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 22:39:43,309 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,309 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 22:39:43,311 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 22:39:43,313 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 22:39:43,316 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 22:39:43,317 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 22:39:43,319 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 22:39:43,319 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 22:39:43,330 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 22:39:43,334 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 22:39:43,336 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 22:39:43,337 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 22:39:43,337 main DEBUG createAppenders(={Console}) 2024-11-17 22:39:43,338 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-17 22:39:43,339 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 22:39:43,339 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-17 22:39:43,340 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 22:39:43,340 main DEBUG OutputStream closed 2024-11-17 22:39:43,341 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 22:39:43,341 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 22:39:43,342 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-17 22:39:43,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 22:39:43,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 22:39:43,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 22:39:43,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 22:39:43,449 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 22:39:43,449 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 22:39:43,450 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 22:39:43,450 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 22:39:43,451 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 22:39:43,451 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 22:39:43,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 22:39:43,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 22:39:43,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 22:39:43,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 22:39:43,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 22:39:43,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 22:39:43,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 22:39:43,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 22:39:43,457 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 22:39:43,458 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-17 22:39:43,458 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 22:39:43,459 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-17T22:39:43,812 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b 2024-11-17 22:39:43,816 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 22:39:43,817 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T22:39:43,828 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-17T22:39:43,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T22:39:43,900 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/cluster_de41a222-fb27-5a57-9fde-672530362f70, deleteOnExit=true 2024-11-17T22:39:43,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T22:39:43,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/test.cache.data in system properties and HBase conf 2024-11-17T22:39:43,904 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T22:39:43,905 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/hadoop.log.dir in system properties and HBase conf 2024-11-17T22:39:43,907 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T22:39:43,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T22:39:43,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T22:39:44,040 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T22:39:44,182 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T22:39:44,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T22:39:44,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T22:39:44,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T22:39:44,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T22:39:44,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T22:39:44,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T22:39:44,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T22:39:44,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T22:39:44,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T22:39:44,195 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/nfs.dump.dir in system properties and HBase conf 2024-11-17T22:39:44,195 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/java.io.tmpdir in system properties and HBase conf 2024-11-17T22:39:44,196 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T22:39:44,196 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T22:39:44,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T22:39:45,134 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T22:39:45,248 INFO [Time-limited test {}] log.Log(170): Logging initialized @2940ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T22:39:45,342 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T22:39:45,422 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T22:39:45,453 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T22:39:45,454 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T22:39:45,455 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T22:39:45,470 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T22:39:45,475 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/hadoop.log.dir/,AVAILABLE} 2024-11-17T22:39:45,476 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T22:39:45,701 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/java.io.tmpdir/jetty-localhost-37325-hadoop-hdfs-3_4_1-tests_jar-_-any-12815487650499027548/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T22:39:45,720 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37325} 2024-11-17T22:39:45,720 INFO [Time-limited test {}] server.Server(415): Started @3413ms 2024-11-17T22:39:46,144 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T22:39:46,153 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T22:39:46,154 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T22:39:46,154 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T22:39:46,155 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T22:39:46,156 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/hadoop.log.dir/,AVAILABLE} 2024-11-17T22:39:46,157 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T22:39:46,280 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/java.io.tmpdir/jetty-localhost-36179-hadoop-hdfs-3_4_1-tests_jar-_-any-2730440903237339808/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T22:39:46,281 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:36179} 2024-11-17T22:39:46,281 INFO [Time-limited test {}] server.Server(415): Started @3974ms 2024-11-17T22:39:46,336 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T22:39:46,974 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/cluster_de41a222-fb27-5a57-9fde-672530362f70/data/data1/current/BP-885076362-172.17.0.2-1731883184876/current, will proceed with Du for space computation calculation, 2024-11-17T22:39:46,986 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/cluster_de41a222-fb27-5a57-9fde-672530362f70/data/data2/current/BP-885076362-172.17.0.2-1731883184876/current, will proceed with Du for space computation calculation, 2024-11-17T22:39:47,081 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T22:39:47,167 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe0d0c2d6b7c01dcb with lease ID 0x3a4444d272130759: Processing first storage report for DS-8d82270f-e0a0-47f0-95f7-901dee2cce90 from datanode DatanodeRegistration(127.0.0.1:39965, datanodeUuid=22b3c91f-f4f7-4331-bfee-028d2ec13cf1, infoPort=37509, infoSecurePort=0, ipcPort=44089, storageInfo=lv=-57;cid=testClusterID;nsid=973417641;c=1731883184876) 2024-11-17T22:39:47,169 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe0d0c2d6b7c01dcb with lease ID 0x3a4444d272130759: from storage DS-8d82270f-e0a0-47f0-95f7-901dee2cce90 node DatanodeRegistration(127.0.0.1:39965, datanodeUuid=22b3c91f-f4f7-4331-bfee-028d2ec13cf1, infoPort=37509, infoSecurePort=0, ipcPort=44089, storageInfo=lv=-57;cid=testClusterID;nsid=973417641;c=1731883184876), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-17T22:39:47,169 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe0d0c2d6b7c01dcb with lease ID 0x3a4444d272130759: Processing first storage report for DS-237d9c79-a147-4676-9890-4bc964b2df44 from datanode DatanodeRegistration(127.0.0.1:39965, datanodeUuid=22b3c91f-f4f7-4331-bfee-028d2ec13cf1, infoPort=37509, infoSecurePort=0, ipcPort=44089, storageInfo=lv=-57;cid=testClusterID;nsid=973417641;c=1731883184876) 2024-11-17T22:39:47,170 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe0d0c2d6b7c01dcb with lease ID 0x3a4444d272130759: from storage DS-237d9c79-a147-4676-9890-4bc964b2df44 node DatanodeRegistration(127.0.0.1:39965, datanodeUuid=22b3c91f-f4f7-4331-bfee-028d2ec13cf1, infoPort=37509, infoSecurePort=0, ipcPort=44089, storageInfo=lv=-57;cid=testClusterID;nsid=973417641;c=1731883184876), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T22:39:47,248 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b 2024-11-17T22:39:47,344 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/cluster_de41a222-fb27-5a57-9fde-672530362f70/zookeeper_0, clientPort=63234, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/cluster_de41a222-fb27-5a57-9fde-672530362f70/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/cluster_de41a222-fb27-5a57-9fde-672530362f70/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T22:39:47,363 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63234 2024-11-17T22:39:47,380 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T22:39:47,385 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T22:39:47,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741825_1001 (size=7) 2024-11-17T22:39:48,074 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71 with version=8 2024-11-17T22:39:48,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/hbase-staging 2024-11-17T22:39:48,178 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T22:39:48,408 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1a6e40b21a48:0 server-side Connection retries=45 2024-11-17T22:39:48,420 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T22:39:48,420 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T22:39:48,426 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T22:39:48,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T22:39:48,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T22:39:48,564 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T22:39:48,622 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T22:39:48,631 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T22:39:48,635 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T22:39:48,660 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1547 (auto-detected) 2024-11-17T22:39:48,662 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T22:39:48,681 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45671 2024-11-17T22:39:48,704 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45671 connecting to ZooKeeper ensemble=127.0.0.1:63234 2024-11-17T22:39:48,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:456710x0, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T22:39:48,736 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45671-0x1004fd798e10000 connected 2024-11-17T22:39:48,765 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T22:39:48,768 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T22:39:48,778 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T22:39:48,783 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71, hbase.cluster.distributed=false 2024-11-17T22:39:48,811 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T22:39:48,816 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45671 2024-11-17T22:39:48,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45671 2024-11-17T22:39:48,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45671 2024-11-17T22:39:48,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45671 2024-11-17T22:39:48,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45671 2024-11-17T22:39:48,945 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1a6e40b21a48:0 server-side Connection retries=45 2024-11-17T22:39:48,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T22:39:48,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T22:39:48,948 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T22:39:48,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T22:39:48,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T22:39:48,952 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T22:39:48,956 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T22:39:48,957 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43011 2024-11-17T22:39:48,960 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43011 connecting to ZooKeeper ensemble=127.0.0.1:63234 2024-11-17T22:39:48,962 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T22:39:48,967 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T22:39:48,975 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430110x0, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T22:39:48,976 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43011-0x1004fd798e10001 connected 2024-11-17T22:39:48,977 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T22:39:48,982 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T22:39:48,991 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T22:39:48,994 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T22:39:48,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T22:39:49,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43011 2024-11-17T22:39:49,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43011 2024-11-17T22:39:49,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43011 2024-11-17T22:39:49,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43011 2024-11-17T22:39:49,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43011 2024-11-17T22:39:49,023 DEBUG [M:0;1a6e40b21a48:45671 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1a6e40b21a48:45671 2024-11-17T22:39:49,025 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1a6e40b21a48,45671,1731883188234 2024-11-17T22:39:49,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T22:39:49,032 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T22:39:49,034 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1a6e40b21a48,45671,1731883188234 2024-11-17T22:39:49,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:49,060 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T22:39:49,060 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:49,061 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T22:39:49,063 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1a6e40b21a48,45671,1731883188234 from backup master directory 2024-11-17T22:39:49,065 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T22:39:49,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1a6e40b21a48,45671,1731883188234 2024-11-17T22:39:49,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T22:39:49,066 WARN [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T22:39:49,066 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1a6e40b21a48,45671,1731883188234 2024-11-17T22:39:49,068 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T22:39:49,070 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T22:39:49,125 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/hbase.id] with ID: 6a900e0c-d2b7-4958-a2a4-2ef9619222c7 2024-11-17T22:39:49,125 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/.tmp/hbase.id 2024-11-17T22:39:49,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741826_1002 (size=42) 2024-11-17T22:39:49,538 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/.tmp/hbase.id]:[hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/hbase.id] 2024-11-17T22:39:49,591 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T22:39:49,597 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T22:39:49,616 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-17T22:39:49,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:49,620 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:49,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741827_1003 (size=196) 2024-11-17T22:39:50,052 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T22:39:50,055 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T22:39:50,072 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:50,077 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T22:39:50,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741828_1004 (size=1189) 2024-11-17T22:39:50,525 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store 2024-11-17T22:39:50,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741829_1005 (size=34) 2024-11-17T22:39:50,951 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T22:39:50,954 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:39:50,956 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T22:39:50,956 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T22:39:50,957 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T22:39:50,958 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T22:39:50,959 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T22:39:50,959 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T22:39:50,961 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731883190956Disabling compacts and flushes for region at 1731883190956Disabling writes for close at 1731883190959 (+3 ms)Writing region close event to WAL at 1731883190959Closed at 1731883190959 2024-11-17T22:39:50,964 WARN [master/1a6e40b21a48:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/.initializing 2024-11-17T22:39:50,965 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/WALs/1a6e40b21a48,45671,1731883188234 2024-11-17T22:39:50,976 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T22:39:50,992 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a6e40b21a48%2C45671%2C1731883188234, suffix=, logDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/WALs/1a6e40b21a48,45671,1731883188234, archiveDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/oldWALs, maxLogs=10 2024-11-17T22:39:51,013 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/WALs/1a6e40b21a48,45671,1731883188234/1a6e40b21a48%2C45671%2C1731883188234.1731883190997, exclude list is [], retry=0 2024-11-17T22:39:51,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39965,DS-8d82270f-e0a0-47f0-95f7-901dee2cce90,DISK] 2024-11-17T22:39:51,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T22:39:51,064 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/WALs/1a6e40b21a48,45671,1731883188234/1a6e40b21a48%2C45671%2C1731883188234.1731883190997 2024-11-17T22:39:51,065 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37509:37509)] 2024-11-17T22:39:51,065 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T22:39:51,066 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:39:51,068 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,069 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T22:39:51,130 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:51,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T22:39:51,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T22:39:51,138 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:51,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:39:51,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T22:39:51,143 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:51,145 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:39:51,145 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,148 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T22:39:51,148 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:51,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:39:51,149 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,154 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,155 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,161 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,162 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,166 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T22:39:51,171 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T22:39:51,177 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T22:39:51,179 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61105100, jitterRate=-0.08946305513381958}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T22:39:51,188 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731883191080Initializing all the Stores at 1731883191082 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731883191082Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883191083 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883191083Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883191084 (+1 ms)Cleaning up temporary data from old regions at 1731883191162 (+78 ms)Region opened successfully at 1731883191188 (+26 ms) 2024-11-17T22:39:51,190 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T22:39:51,224 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe9447b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a6e40b21a48/172.17.0.2:0 2024-11-17T22:39:51,252 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T22:39:51,262 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T22:39:51,262 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T22:39:51,265 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T22:39:51,266 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T22:39:51,270 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T22:39:51,271 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T22:39:51,296 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T22:39:51,308 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T22:39:51,309 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T22:39:51,312 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T22:39:51,314 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T22:39:51,315 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T22:39:51,317 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T22:39:51,321 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T22:39:51,322 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T22:39:51,324 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T22:39:51,325 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T22:39:51,342 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T22:39:51,343 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T22:39:51,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T22:39:51,347 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T22:39:51,347 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:51,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:51,349 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1a6e40b21a48,45671,1731883188234, sessionid=0x1004fd798e10000, setting cluster-up flag (Was=false) 2024-11-17T22:39:51,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:51,363 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:51,368 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T22:39:51,370 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a6e40b21a48,45671,1731883188234 2024-11-17T22:39:51,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:51,375 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:51,380 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T22:39:51,382 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a6e40b21a48,45671,1731883188234 2024-11-17T22:39:51,389 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T22:39:51,415 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(746): ClusterId : 6a900e0c-d2b7-4958-a2a4-2ef9619222c7 2024-11-17T22:39:51,418 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T22:39:51,423 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T22:39:51,423 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T22:39:51,426 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T22:39:51,427 DEBUG [RS:0;1a6e40b21a48:43011 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa37fd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a6e40b21a48/172.17.0.2:0 2024-11-17T22:39:51,444 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1a6e40b21a48:43011 2024-11-17T22:39:51,448 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T22:39:51,448 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T22:39:51,448 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T22:39:51,451 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a6e40b21a48,45671,1731883188234 with port=43011, startcode=1731883188895 2024-11-17T22:39:51,466 DEBUG [RS:0;1a6e40b21a48:43011 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T22:39:51,472 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T22:39:51,485 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T22:39:51,496 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T22:39:51,503 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1a6e40b21a48,45671,1731883188234 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T22:39:51,509 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38713, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T22:39:51,517 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45671 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:51,520 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1a6e40b21a48:0, corePoolSize=5, maxPoolSize=5 2024-11-17T22:39:51,520 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1a6e40b21a48:0, corePoolSize=5, maxPoolSize=5 2024-11-17T22:39:51,520 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1a6e40b21a48:0, corePoolSize=5, maxPoolSize=5 2024-11-17T22:39:51,521 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1a6e40b21a48:0, corePoolSize=5, maxPoolSize=5 2024-11-17T22:39:51,521 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1a6e40b21a48:0, corePoolSize=10, maxPoolSize=10 2024-11-17T22:39:51,521 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,522 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1a6e40b21a48:0, corePoolSize=2, maxPoolSize=2 2024-11-17T22:39:51,522 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,532 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T22:39:51,533 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T22:39:51,538 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:51,539 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731883221539 2024-11-17T22:39:51,539 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T22:39:51,541 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T22:39:51,543 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T22:39:51,547 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T22:39:51,548 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T22:39:51,548 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T22:39:51,548 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T22:39:51,551 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T22:39:51,551 WARN [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T22:39:51,553 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,561 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T22:39:51,563 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T22:39:51,563 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T22:39:51,565 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T22:39:51,566 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T22:39:51,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741831_1007 (size=1321) 2024-11-17T22:39:51,570 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T22:39:51,571 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71 2024-11-17T22:39:51,573 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1a6e40b21a48:0:becomeActiveMaster-HFileCleaner.large.0-1731883191568,5,FailOnTimeoutGroup] 2024-11-17T22:39:51,577 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1a6e40b21a48:0:becomeActiveMaster-HFileCleaner.small.0-1731883191573,5,FailOnTimeoutGroup] 2024-11-17T22:39:51,577 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,578 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T22:39:51,579 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,580 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741832_1008 (size=32) 2024-11-17T22:39:51,597 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:39:51,600 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T22:39:51,604 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T22:39:51,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:51,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T22:39:51,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T22:39:51,609 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T22:39:51,609 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:51,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T22:39:51,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T22:39:51,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T22:39:51,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:51,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T22:39:51,616 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T22:39:51,619 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T22:39:51,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:51,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T22:39:51,621 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T22:39:51,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740 2024-11-17T22:39:51,630 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740 2024-11-17T22:39:51,635 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T22:39:51,635 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T22:39:51,636 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T22:39:51,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T22:39:51,650 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T22:39:51,653 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a6e40b21a48,45671,1731883188234 with port=43011, startcode=1731883188895 2024-11-17T22:39:51,655 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63404987, jitterRate=-0.055192068219184875}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T22:39:51,660 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731883191597Initializing all the Stores at 1731883191600 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731883191600Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731883191600Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883191600Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731883191600Cleaning up temporary data from old regions at 1731883191635 (+35 ms)Region opened successfully at 1731883191660 (+25 ms) 2024-11-17T22:39:51,660 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T22:39:51,660 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T22:39:51,660 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45671 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:51,661 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T22:39:51,661 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T22:39:51,661 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T22:39:51,662 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T22:39:51,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731883191660Disabling compacts and flushes for region at 1731883191660Disabling writes for close at 1731883191661 (+1 ms)Writing region close event to WAL at 1731883191662 (+1 ms)Closed at 1731883191662 2024-11-17T22:39:51,663 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45671 {}] master.ServerManager(517): Registering regionserver=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:51,668 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T22:39:51,668 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T22:39:51,672 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71 2024-11-17T22:39:51,672 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41421 2024-11-17T22:39:51,672 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T22:39:51,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T22:39:51,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T22:39:51,677 DEBUG [RS:0;1a6e40b21a48:43011 {}] zookeeper.ZKUtil(111): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:51,678 WARN [RS:0;1a6e40b21a48:43011 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T22:39:51,678 INFO [RS:0;1a6e40b21a48:43011 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T22:39:51,678 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/WALs/1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:51,681 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a6e40b21a48,43011,1731883188895] 2024-11-17T22:39:51,688 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T22:39:51,691 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T22:39:51,708 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T22:39:51,727 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T22:39:51,733 INFO [RS:0;1a6e40b21a48:43011 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T22:39:51,733 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,735 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T22:39:51,743 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T22:39:51,745 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,745 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,746 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,746 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,746 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,747 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,747 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a6e40b21a48:0, corePoolSize=2, maxPoolSize=2 2024-11-17T22:39:51,747 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,747 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,747 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,748 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,748 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,748 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a6e40b21a48:0, corePoolSize=1, maxPoolSize=1 2024-11-17T22:39:51,748 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a6e40b21a48:0, corePoolSize=3, maxPoolSize=3 2024-11-17T22:39:51,748 DEBUG [RS:0;1a6e40b21a48:43011 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0, corePoolSize=3, maxPoolSize=3 2024-11-17T22:39:51,750 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,750 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,751 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,751 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,751 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,751 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a6e40b21a48,43011,1731883188895-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T22:39:51,773 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T22:39:51,775 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a6e40b21a48,43011,1731883188895-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,775 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,775 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.Replication(171): 1a6e40b21a48,43011,1731883188895 started 2024-11-17T22:39:51,798 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:51,799 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1482): Serving as 1a6e40b21a48,43011,1731883188895, RpcServer on 1a6e40b21a48/172.17.0.2:43011, sessionid=0x1004fd798e10001 2024-11-17T22:39:51,800 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T22:39:51,800 DEBUG [RS:0;1a6e40b21a48:43011 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:51,800 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a6e40b21a48,43011,1731883188895' 2024-11-17T22:39:51,801 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T22:39:51,802 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T22:39:51,803 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T22:39:51,803 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T22:39:51,803 DEBUG [RS:0;1a6e40b21a48:43011 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:51,804 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a6e40b21a48,43011,1731883188895' 2024-11-17T22:39:51,804 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T22:39:51,805 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T22:39:51,805 DEBUG [RS:0;1a6e40b21a48:43011 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T22:39:51,806 INFO [RS:0;1a6e40b21a48:43011 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T22:39:51,806 INFO [RS:0;1a6e40b21a48:43011 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T22:39:51,843 WARN [1a6e40b21a48:45671 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T22:39:51,916 INFO [RS:0;1a6e40b21a48:43011 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T22:39:51,923 INFO [RS:0;1a6e40b21a48:43011 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a6e40b21a48%2C43011%2C1731883188895, suffix=, logDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/WALs/1a6e40b21a48,43011,1731883188895, archiveDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/oldWALs, maxLogs=32 2024-11-17T22:39:51,941 DEBUG [RS:0;1a6e40b21a48:43011 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/WALs/1a6e40b21a48,43011,1731883188895/1a6e40b21a48%2C43011%2C1731883188895.1731883191926, exclude list is [], retry=0 2024-11-17T22:39:51,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39965,DS-8d82270f-e0a0-47f0-95f7-901dee2cce90,DISK] 2024-11-17T22:39:51,950 INFO [RS:0;1a6e40b21a48:43011 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/WALs/1a6e40b21a48,43011,1731883188895/1a6e40b21a48%2C43011%2C1731883188895.1731883191926 2024-11-17T22:39:51,950 DEBUG [RS:0;1a6e40b21a48:43011 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37509:37509)] 2024-11-17T22:39:52,096 DEBUG [1a6e40b21a48:45671 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-17T22:39:52,108 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:52,114 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a6e40b21a48,43011,1731883188895, state=OPENING 2024-11-17T22:39:52,118 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T22:39:52,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:52,119 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:39:52,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T22:39:52,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T22:39:52,122 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T22:39:52,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:39:52,298 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T22:39:52,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52329, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T22:39:52,312 INFO [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T22:39:52,312 INFO [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T22:39:52,313 INFO [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T22:39:52,316 INFO [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a6e40b21a48%2C43011%2C1731883188895.meta, suffix=.meta, logDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/WALs/1a6e40b21a48,43011,1731883188895, archiveDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/oldWALs, maxLogs=32 2024-11-17T22:39:52,331 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/WALs/1a6e40b21a48,43011,1731883188895/1a6e40b21a48%2C43011%2C1731883188895.meta.1731883192318.meta, exclude list is [], retry=0 2024-11-17T22:39:52,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39965,DS-8d82270f-e0a0-47f0-95f7-901dee2cce90,DISK] 2024-11-17T22:39:52,337 INFO [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/WALs/1a6e40b21a48,43011,1731883188895/1a6e40b21a48%2C43011%2C1731883188895.meta.1731883192318.meta 2024-11-17T22:39:52,338 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37509:37509)] 2024-11-17T22:39:52,338 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T22:39:52,340 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T22:39:52,342 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T22:39:52,346 INFO [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T22:39:52,350 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T22:39:52,350 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:39:52,351 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T22:39:52,351 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T22:39:52,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T22:39:52,355 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T22:39:52,355 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:52,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T22:39:52,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T22:39:52,358 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T22:39:52,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:52,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T22:39:52,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T22:39:52,360 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T22:39:52,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:52,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T22:39:52,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T22:39:52,362 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T22:39:52,362 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:52,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T22:39:52,364 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T22:39:52,365 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740 2024-11-17T22:39:52,368 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740 2024-11-17T22:39:52,370 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T22:39:52,370 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T22:39:52,371 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T22:39:52,374 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T22:39:52,375 INFO [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63190810, jitterRate=-0.058383554220199585}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T22:39:52,375 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T22:39:52,377 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731883192351Writing region info on filesystem at 1731883192351Initializing all the Stores at 1731883192353 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731883192353Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731883192353Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883192353Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731883192353Cleaning up temporary data from old regions at 1731883192370 (+17 ms)Running coprocessor post-open hooks at 1731883192375 (+5 ms)Region opened successfully at 1731883192377 (+2 ms) 2024-11-17T22:39:52,383 INFO [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731883192291 2024-11-17T22:39:52,394 DEBUG [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T22:39:52,394 INFO [RS_OPEN_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T22:39:52,396 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:52,398 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a6e40b21a48,43011,1731883188895, state=OPEN 2024-11-17T22:39:52,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T22:39:52,401 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T22:39:52,401 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T22:39:52,402 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T22:39:52,402 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:52,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T22:39:52,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a6e40b21a48,43011,1731883188895 in 279 msec 2024-11-17T22:39:52,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T22:39:52,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 733 msec 2024-11-17T22:39:52,416 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T22:39:52,416 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T22:39:52,434 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:52,435 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:52,452 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:52,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58419, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:52,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0540 sec 2024-11-17T22:39:52,474 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731883192474, completionTime=-1 2024-11-17T22:39:52,477 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-17T22:39:52,477 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T22:39:52,506 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-17T22:39:52,506 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731883252506 2024-11-17T22:39:52,506 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731883312506 2024-11-17T22:39:52,506 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-17T22:39:52,508 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a6e40b21a48,45671,1731883188234-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:52,509 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a6e40b21a48,45671,1731883188234-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:52,509 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a6e40b21a48,45671,1731883188234-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:52,510 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1a6e40b21a48:45671, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:52,510 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:52,511 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:52,516 DEBUG [master/1a6e40b21a48:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T22:39:52,536 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.469sec 2024-11-17T22:39:52,537 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T22:39:52,539 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T22:39:52,540 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T22:39:52,540 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T22:39:52,540 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T22:39:52,541 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a6e40b21a48,45671,1731883188234-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T22:39:52,542 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a6e40b21a48,45671,1731883188234-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T22:39:52,549 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T22:39:52,550 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T22:39:52,551 INFO [master/1a6e40b21a48:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a6e40b21a48,45671,1731883188234-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T22:39:52,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:52,629 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T22:39:52,629 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T22:39:52,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:39:52,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:39:52,641 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:39:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:39:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:39:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5803a924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:39:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:39:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:39:52,650 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:39:52,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:52,653 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:52,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:52,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1a6e40b21a48,45671,1731883188234 2024-11-17T22:39:52,695 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=311, ProcessCount=11, AvailableMemoryMB=5299 2024-11-17T22:39:52,704 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T22:39:52,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a6e40b21a48,45671,1731883188234 2024-11-17T22:39:52,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5164c17f 2024-11-17T22:39:52,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T22:39:52,715 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T22:39:52,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T22:39:52,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T22:39:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T22:39:52,731 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T22:39:52,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-17T22:39:52,733 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:52,736 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T22:39:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T22:39:52,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741835_1011 (size=959) 2024-11-17T22:39:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T22:39:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T22:39:53,158 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71 2024-11-17T22:39:53,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741836_1012 (size=53) 2024-11-17T22:39:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T22:39:53,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:39:53,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing dde7707c5ff08c6cad672aef50110927, disabling compactions & flushes 2024-11-17T22:39:53,571 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:53,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:53,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. after waiting 0 ms 2024-11-17T22:39:53,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:53,571 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:53,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for dde7707c5ff08c6cad672aef50110927: Waiting for close lock at 1731883193571Disabling compacts and flushes for region at 1731883193571Disabling writes for close at 1731883193571Writing region close event to WAL at 1731883193571Closed at 1731883193571 2024-11-17T22:39:53,575 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T22:39:53,582 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731883193575"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731883193575"}]},"ts":"1731883193575"} 2024-11-17T22:39:53,586 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T22:39:53,589 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T22:39:53,592 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883193589"}]},"ts":"1731883193589"} 2024-11-17T22:39:53,597 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T22:39:53,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde7707c5ff08c6cad672aef50110927, ASSIGN}] 2024-11-17T22:39:53,603 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde7707c5ff08c6cad672aef50110927, ASSIGN 2024-11-17T22:39:53,605 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde7707c5ff08c6cad672aef50110927, ASSIGN; state=OFFLINE, location=1a6e40b21a48,43011,1731883188895; forceNewPlan=false, retain=false 2024-11-17T22:39:53,757 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dde7707c5ff08c6cad672aef50110927, regionState=OPENING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:53,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde7707c5ff08c6cad672aef50110927, ASSIGN because future has completed 2024-11-17T22:39:53,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:39:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T22:39:53,929 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:53,929 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} 2024-11-17T22:39:53,930 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,930 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:39:53,930 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,930 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,932 INFO [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,935 INFO [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde7707c5ff08c6cad672aef50110927 columnFamilyName A 2024-11-17T22:39:53,935 DEBUG [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:53,936 INFO [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] regionserver.HStore(327): Store=dde7707c5ff08c6cad672aef50110927/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:39:53,936 INFO [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,938 INFO [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde7707c5ff08c6cad672aef50110927 columnFamilyName B 2024-11-17T22:39:53,938 DEBUG [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:53,939 INFO [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] regionserver.HStore(327): Store=dde7707c5ff08c6cad672aef50110927/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:39:53,939 INFO [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,941 INFO [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde7707c5ff08c6cad672aef50110927 columnFamilyName C 2024-11-17T22:39:53,942 DEBUG [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:39:53,942 INFO [StoreOpener-dde7707c5ff08c6cad672aef50110927-1 {}] regionserver.HStore(327): Store=dde7707c5ff08c6cad672aef50110927/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:39:53,943 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,944 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,945 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,947 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,947 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,948 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T22:39:53,951 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,955 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T22:39:53,956 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened dde7707c5ff08c6cad672aef50110927; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59876512, jitterRate=-0.10777044296264648}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T22:39:53,957 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:53,958 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for dde7707c5ff08c6cad672aef50110927: Running coprocessor pre-open hook at 1731883193930Writing region info on filesystem at 1731883193930Initializing all the Stores at 1731883193932 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883193932Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883193932Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883193932Cleaning up temporary data from old regions at 1731883193947 (+15 ms)Running coprocessor post-open hooks at 1731883193957 (+10 ms)Region opened successfully at 1731883193958 (+1 ms) 2024-11-17T22:39:53,963 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., pid=6, masterSystemTime=1731883193918 2024-11-17T22:39:53,967 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:53,968 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:53,969 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dde7707c5ff08c6cad672aef50110927, regionState=OPEN, openSeqNum=2, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:53,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:39:53,978 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45671 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1a6e40b21a48,43011,1731883188895, table=TestAcidGuarantees, region=dde7707c5ff08c6cad672aef50110927. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T22:39:53,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T22:39:53,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 in 225 msec 2024-11-17T22:39:54,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T22:39:54,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde7707c5ff08c6cad672aef50110927, ASSIGN in 397 msec 2024-11-17T22:39:54,004 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T22:39:54,004 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883194004"}]},"ts":"1731883194004"} 2024-11-17T22:39:54,009 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T22:39:54,012 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T22:39:54,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2900 sec 2024-11-17T22:39:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T22:39:54,889 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:39:54,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:54,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:39:54,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:39:54,901 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:39:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:39:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:39:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63473470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:39:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:39:54,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:39:54,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:39:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302d7c14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:54,909 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:54,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:54,917 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T22:39:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a6e40b21a48,45671,1731883188234 2024-11-17T22:39:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@d32c91 2024-11-17T22:39:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T22:39:54,925 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T22:39:54,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:54,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:39:54,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:39:54,932 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:39:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:39:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:39:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ffac79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:39:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:39:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:39:54,939 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:39:54,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0184e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:54,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:54,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:39:54,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:39:54,963 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:39:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:39:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:39:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cbf42c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:39:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:39:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:39:54,967 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:39:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cb0f4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:54,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:54,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:39:54,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:39:54,982 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:39:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:39:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:39:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603c95bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:39:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:39:54,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:39:54,996 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:39:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6b9e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:39:55,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:39:55,006 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:39:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:39:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:39:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c9e84c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:39:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:39:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:39:55,019 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:39:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b8091e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:39:55,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:39:55,031 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:39:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:39:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:39:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f99b884, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:39:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:39:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:39:55,037 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:39:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75abeb9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:39:55,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:39:55,044 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:39:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:39:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:39:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598c3d89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:39:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:39:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:39:55,048 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:39:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec99be6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:39:55,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:39:55,053 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:39:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:39:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:39:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6fe38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:39:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:39:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:39:55,057 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:39:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60da3efe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:39:55,060 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:39:55,062 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:39:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:39:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:39:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9487f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:39:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:39:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:39:55,066 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:39:55,068 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff7d382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,074 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:39:55,074 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:39:55,076 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:39:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:39:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:39:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56900a9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:39:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:39:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:39:55,079 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:39:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51ceab7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:39:55,096 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:55,099 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:55,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:55,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:55,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:55,104 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:55,104 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:55,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:39:55,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:55,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:39:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:39:55,120 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:55,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:55,126 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:55,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:39:55,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:39:55,138 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:55,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:55,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:39:55,158 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:55,163 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:39:55,174 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:55,176 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:39:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:39:55,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:39:55,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:39:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:39:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:39:55,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:39:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-17T22:39:55,226 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:39:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T22:39:55,229 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:39:55,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:39:55,241 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:39:55,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:39:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T22:39:55,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/2fffadd4ec124ffc832f3062aa114f63 is 50, key is test_row_0/A:col10/1731883195237/Put/seqid=0 2024-11-17T22:39:55,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T22:39:55,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:55,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:55,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:55,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:55,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:55,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883255423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741837_1013 (size=12001) 2024-11-17T22:39:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:55,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883255435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883255438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883255443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883255445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T22:39:55,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883255568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883255582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883255584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883255584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883255584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T22:39:55,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:55,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:55,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:55,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:55,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:55,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883255777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T22:39:55,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883255800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:55,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:55,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:55,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883255800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:55,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883255801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883255801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T22:39:55,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/2fffadd4ec124ffc832f3062aa114f63 2024-11-17T22:39:55,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T22:39:55,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:55,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:55,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:55,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:55,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/a9f4487f660340fcac9d5df9d363c99b is 50, key is test_row_0/B:col10/1731883195237/Put/seqid=0 2024-11-17T22:39:56,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741838_1014 (size=12001) 2024-11-17T22:39:56,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/a9f4487f660340fcac9d5df9d363c99b 2024-11-17T22:39:56,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883256095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:56,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T22:39:56,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:56,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:56,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:56,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883256118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:56,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:56,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883256119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883256119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:56,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883256128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/b91ec837a302448dbfed0312fa31f395 is 50, key is test_row_0/C:col10/1731883195237/Put/seqid=0 2024-11-17T22:39:56,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741839_1015 (size=12001) 2024-11-17T22:39:56,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T22:39:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:56,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T22:39:56,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T22:39:56,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:56,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:56,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:56,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/b91ec837a302448dbfed0312fa31f395 2024-11-17T22:39:56,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883256608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:56,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T22:39:56,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:56,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:56,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:56,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:56,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/2fffadd4ec124ffc832f3062aa114f63 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2fffadd4ec124ffc832f3062aa114f63 2024-11-17T22:39:56,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883256638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:56,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883256638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:56,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883256639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:56,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:56,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2fffadd4ec124ffc832f3062aa114f63, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T22:39:56,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883256647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:56,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:56,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-17T22:39:56,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/a9f4487f660340fcac9d5df9d363c99b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/a9f4487f660340fcac9d5df9d363c99b 2024-11-17T22:39:56,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/a9f4487f660340fcac9d5df9d363c99b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T22:39:56,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/b91ec837a302448dbfed0312fa31f395 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b91ec837a302448dbfed0312fa31f395 2024-11-17T22:39:56,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b91ec837a302448dbfed0312fa31f395, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T22:39:56,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for dde7707c5ff08c6cad672aef50110927 in 1458ms, sequenceid=13, compaction requested=false 2024-11-17T22:39:56,714 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T22:39:56,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:56,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T22:39:56,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:56,774 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:39:56,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/741c2e376f5348528da06b9e89aeb878 is 50, key is test_row_0/A:col10/1731883195425/Put/seqid=0 2024-11-17T22:39:56,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741840_1016 (size=12001) 2024-11-17T22:39:56,817 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/741c2e376f5348528da06b9e89aeb878 2024-11-17T22:39:56,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/ad9e879fb4e9430690f0fb670b9b781b is 50, key is test_row_0/B:col10/1731883195425/Put/seqid=0 2024-11-17T22:39:56,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741841_1017 (size=12001) 2024-11-17T22:39:56,880 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/ad9e879fb4e9430690f0fb670b9b781b 2024-11-17T22:39:56,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/5bc58d5f84d24958a8551d32a3b017d2 is 50, key is test_row_0/C:col10/1731883195425/Put/seqid=0 2024-11-17T22:39:56,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741842_1018 (size=12001) 2024-11-17T22:39:56,950 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/5bc58d5f84d24958a8551d32a3b017d2 2024-11-17T22:39:56,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/741c2e376f5348528da06b9e89aeb878 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/741c2e376f5348528da06b9e89aeb878 2024-11-17T22:39:56,982 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/741c2e376f5348528da06b9e89aeb878, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T22:39:56,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/ad9e879fb4e9430690f0fb670b9b781b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ad9e879fb4e9430690f0fb670b9b781b 2024-11-17T22:39:57,005 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ad9e879fb4e9430690f0fb670b9b781b, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T22:39:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/5bc58d5f84d24958a8551d32a3b017d2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/5bc58d5f84d24958a8551d32a3b017d2 2024-11-17T22:39:57,028 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/5bc58d5f84d24958a8551d32a3b017d2, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T22:39:57,031 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for dde7707c5ff08c6cad672aef50110927 in 257ms, sequenceid=38, compaction requested=false 2024-11-17T22:39:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:57,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-17T22:39:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-17T22:39:57,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T22:39:57,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-11-17T22:39:57,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.8250 sec 2024-11-17T22:39:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T22:39:57,377 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:39:57,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:39:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-17T22:39:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T22:39:57,383 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:39:57,386 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:39:57,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:39:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T22:39:57,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T22:39:57,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:57,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:57,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:57,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-17T22:39:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-17T22:39:57,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-17T22:39:57,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 160 msec 2024-11-17T22:39:57,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 183 msec 2024-11-17T22:39:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:57,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:39:57,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:39:57,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/ae3cc259bcf84049aabd2451a574a984 is 50, key is test_row_0/A:col10/1731883197659/Put/seqid=0 2024-11-17T22:39:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T22:39:57,698 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:39:57,698 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:39:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-17T22:39:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T22:39:57,708 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:39:57,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741843_1019 (size=12001) 2024-11-17T22:39:57,711 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:39:57,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:39:57,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883257728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:57,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883257731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:57,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883257734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:57,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:57,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883257739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883257739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T22:39:57,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883257848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:57,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883257848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:57,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883257859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:57,864 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T22:39:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:57,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T22:39:57,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:57,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:57,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:57,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:57,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:57,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883257861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:57,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883257862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:57,978 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T22:39:58,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T22:39:58,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:58,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T22:39:58,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883258058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883258067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883258068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883258079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883258078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/ae3cc259bcf84049aabd2451a574a984 2024-11-17T22:39:58,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/c220fd4de64d4d4387d8c47750b94383 is 50, key is test_row_0/B:col10/1731883197659/Put/seqid=0 2024-11-17T22:39:58,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T22:39:58,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:58,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741844_1020 (size=12001) 2024-11-17T22:39:58,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/c220fd4de64d4d4387d8c47750b94383 2024-11-17T22:39:58,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/6b57c966629c45ed8346472f00451a63 is 50, key is test_row_0/C:col10/1731883197659/Put/seqid=0 2024-11-17T22:39:58,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741845_1021 (size=12001) 2024-11-17T22:39:58,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/6b57c966629c45ed8346472f00451a63 2024-11-17T22:39:58,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/ae3cc259bcf84049aabd2451a574a984 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/ae3cc259bcf84049aabd2451a574a984 2024-11-17T22:39:58,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/ae3cc259bcf84049aabd2451a574a984, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T22:39:58,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/c220fd4de64d4d4387d8c47750b94383 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/c220fd4de64d4d4387d8c47750b94383 2024-11-17T22:39:58,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/c220fd4de64d4d4387d8c47750b94383, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T22:39:58,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/6b57c966629c45ed8346472f00451a63 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6b57c966629c45ed8346472f00451a63 2024-11-17T22:39:58,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T22:39:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T22:39:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6b57c966629c45ed8346472f00451a63, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T22:39:58,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for dde7707c5ff08c6cad672aef50110927 in 696ms, sequenceid=50, compaction requested=true 2024-11-17T22:39:58,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:58,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:39:58,370 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:39:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:58,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:39:58,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:39:58,372 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:39:58,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:39:58,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:39:58,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:39:58,372 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:39:58,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-17T22:39:58,376 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:39:58,376 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:39:58,378 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/A is initiating minor compaction (all files) 2024-11-17T22:39:58,378 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/B is initiating minor compaction (all files) 2024-11-17T22:39:58,379 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/A in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,379 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/B in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,380 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/a9f4487f660340fcac9d5df9d363c99b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ad9e879fb4e9430690f0fb670b9b781b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/c220fd4de64d4d4387d8c47750b94383] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=35.2 K 2024-11-17T22:39:58,380 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2fffadd4ec124ffc832f3062aa114f63, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/741c2e376f5348528da06b9e89aeb878, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/ae3cc259bcf84049aabd2451a574a984] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=35.2 K 2024-11-17T22:39:58,382 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting a9f4487f660340fcac9d5df9d363c99b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731883195126 2024-11-17T22:39:58,383 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting ad9e879fb4e9430690f0fb670b9b781b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731883195405 2024-11-17T22:39:58,384 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fffadd4ec124ffc832f3062aa114f63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731883195126 2024-11-17T22:39:58,387 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 741c2e376f5348528da06b9e89aeb878, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731883195405 2024-11-17T22:39:58,388 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae3cc259bcf84049aabd2451a574a984, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883197653 2024-11-17T22:39:58,393 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting c220fd4de64d4d4387d8c47750b94383, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883197653 2024-11-17T22:39:58,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/2d60058b31c64ec195d5ca3dce6f3a04 is 50, key is test_row_0/A:col10/1731883197715/Put/seqid=0 2024-11-17T22:39:58,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883258397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883258406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883258415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883258419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883258423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741846_1022 (size=14341) 2024-11-17T22:39:58,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/2d60058b31c64ec195d5ca3dce6f3a04 2024-11-17T22:39:58,476 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#B#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:39:58,476 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#A#compaction#11 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:39:58,477 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/2f44c4a3f9ac47168e2ee79d8d10d995 is 50, key is test_row_0/A:col10/1731883197659/Put/seqid=0 2024-11-17T22:39:58,482 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/482f0dde04084951bab14c9944876b78 is 50, key is test_row_0/B:col10/1731883197659/Put/seqid=0 2024-11-17T22:39:58,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/5cce1f43d4914273b4d2bc31f66d9a99 is 50, key is test_row_0/B:col10/1731883197715/Put/seqid=0 2024-11-17T22:39:58,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T22:39:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741847_1023 (size=12104) 2024-11-17T22:39:58,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883258537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883258538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883258538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883258538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,547 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/2f44c4a3f9ac47168e2ee79d8d10d995 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2f44c4a3f9ac47168e2ee79d8d10d995 2024-11-17T22:39:58,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741848_1024 (size=12104) 2024-11-17T22:39:58,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741849_1025 (size=12001) 2024-11-17T22:39:58,584 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/482f0dde04084951bab14c9944876b78 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/482f0dde04084951bab14c9944876b78 2024-11-17T22:39:58,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/5cce1f43d4914273b4d2bc31f66d9a99 2024-11-17T22:39:58,595 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/A of dde7707c5ff08c6cad672aef50110927 into 2f44c4a3f9ac47168e2ee79d8d10d995(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:39:58,596 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:58,604 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/A, priority=13, startTime=1731883198359; duration=0sec 2024-11-17T22:39:58,604 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:39:58,605 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:A 2024-11-17T22:39:58,605 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:39:58,609 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/B of dde7707c5ff08c6cad672aef50110927 into 482f0dde04084951bab14c9944876b78(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:39:58,610 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:58,610 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/B, priority=13, startTime=1731883198371; duration=0sec 2024-11-17T22:39:58,610 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:39:58,610 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:B 2024-11-17T22:39:58,610 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:39:58,610 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/C is initiating minor compaction (all files) 2024-11-17T22:39:58,611 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/C in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,611 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b91ec837a302448dbfed0312fa31f395, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/5bc58d5f84d24958a8551d32a3b017d2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6b57c966629c45ed8346472f00451a63] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=35.2 K 2024-11-17T22:39:58,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/d6a3250dc0bc41a288f61f85063e78dd is 50, key is test_row_0/C:col10/1731883197715/Put/seqid=0 2024-11-17T22:39:58,617 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting b91ec837a302448dbfed0312fa31f395, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731883195126 2024-11-17T22:39:58,619 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T22:39:58,619 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T22:39:58,622 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T22:39:58,622 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T22:39:58,622 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bc58d5f84d24958a8551d32a3b017d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731883195405 2024-11-17T22:39:58,622 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T22:39:58,622 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T22:39:58,624 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T22:39:58,624 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b57c966629c45ed8346472f00451a63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883197653 2024-11-17T22:39:58,624 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T22:39:58,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T22:39:58,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:58,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741850_1026 (size=12001) 2024-11-17T22:39:58,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/d6a3250dc0bc41a288f61f85063e78dd 2024-11-17T22:39:58,674 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#C#compaction#14 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:39:58,675 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/214ee6f0fbcb421bb58a69727f21372e is 50, key is test_row_0/C:col10/1731883197659/Put/seqid=0 2024-11-17T22:39:58,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/2d60058b31c64ec195d5ca3dce6f3a04 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2d60058b31c64ec195d5ca3dce6f3a04 2024-11-17T22:39:58,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2d60058b31c64ec195d5ca3dce6f3a04, entries=200, sequenceid=77, filesize=14.0 K 2024-11-17T22:39:58,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/5cce1f43d4914273b4d2bc31f66d9a99 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/5cce1f43d4914273b4d2bc31f66d9a99 2024-11-17T22:39:58,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/5cce1f43d4914273b4d2bc31f66d9a99, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T22:39:58,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741851_1027 (size=12104) 2024-11-17T22:39:58,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/d6a3250dc0bc41a288f61f85063e78dd as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/d6a3250dc0bc41a288f61f85063e78dd 2024-11-17T22:39:58,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/d6a3250dc0bc41a288f61f85063e78dd, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T22:39:58,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for dde7707c5ff08c6cad672aef50110927 in 369ms, sequenceid=77, compaction requested=false 2024-11-17T22:39:58,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:58,742 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/214ee6f0fbcb421bb58a69727f21372e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/214ee6f0fbcb421bb58a69727f21372e 2024-11-17T22:39:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:58,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:39:58,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:39:58,764 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/C of dde7707c5ff08c6cad672aef50110927 into 214ee6f0fbcb421bb58a69727f21372e(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:39:58,764 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:58,764 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/C, priority=13, startTime=1731883198372; duration=0sec 2024-11-17T22:39:58,764 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:39:58,764 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:C 2024-11-17T22:39:58,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/9c2747e3527d4c11b7c89c57ac7d2243 is 50, key is test_row_0/A:col10/1731883198753/Put/seqid=0 2024-11-17T22:39:58,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741852_1028 (size=12001) 2024-11-17T22:39:58,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/9c2747e3527d4c11b7c89c57ac7d2243 2024-11-17T22:39:58,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T22:39:58,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:58,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/8cf936d1dcb3401a8fa55d95196391fb is 50, key is test_row_0/B:col10/1731883198753/Put/seqid=0 2024-11-17T22:39:58,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883258831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883258839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T22:39:58,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883258843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883258844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741853_1029 (size=12001) 2024-11-17T22:39:58,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/8cf936d1dcb3401a8fa55d95196391fb 2024-11-17T22:39:58,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/f56542f3159940918ea1e8c5ea4a77c0 is 50, key is test_row_0/C:col10/1731883198753/Put/seqid=0 2024-11-17T22:39:58,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741854_1030 (size=12001) 2024-11-17T22:39:58,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/f56542f3159940918ea1e8c5ea4a77c0 2024-11-17T22:39:58,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/9c2747e3527d4c11b7c89c57ac7d2243 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/9c2747e3527d4c11b7c89c57ac7d2243 2024-11-17T22:39:58,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/9c2747e3527d4c11b7c89c57ac7d2243, entries=150, sequenceid=91, filesize=11.7 K 2024-11-17T22:39:58,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/8cf936d1dcb3401a8fa55d95196391fb as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8cf936d1dcb3401a8fa55d95196391fb 2024-11-17T22:39:58,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883258938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8cf936d1dcb3401a8fa55d95196391fb, entries=150, sequenceid=91, filesize=11.7 K 2024-11-17T22:39:58,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883258958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883258958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883258958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/f56542f3159940918ea1e8c5ea4a77c0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f56542f3159940918ea1e8c5ea4a77c0 2024-11-17T22:39:58,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T22:39:58,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:39:58,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883258967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:58,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:58,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:58,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f56542f3159940918ea1e8c5ea4a77c0, entries=150, sequenceid=91, filesize=11.7 K 2024-11-17T22:39:58,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for dde7707c5ff08c6cad672aef50110927 in 230ms, sequenceid=91, compaction requested=true 2024-11-17T22:39:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:58,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:39:58,987 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:39:58,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:39:58,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:39:58,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:39:58,987 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:39:58,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:39:58,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:39:58,991 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:39:58,991 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:39:58,991 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/B is initiating minor compaction (all files) 2024-11-17T22:39:58,991 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/A is initiating minor compaction (all files) 2024-11-17T22:39:58,991 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/B in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,991 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/A in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:58,992 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/482f0dde04084951bab14c9944876b78, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/5cce1f43d4914273b4d2bc31f66d9a99, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8cf936d1dcb3401a8fa55d95196391fb] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=35.3 K 2024-11-17T22:39:58,992 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2f44c4a3f9ac47168e2ee79d8d10d995, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2d60058b31c64ec195d5ca3dce6f3a04, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/9c2747e3527d4c11b7c89c57ac7d2243] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=37.5 K 2024-11-17T22:39:58,993 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f44c4a3f9ac47168e2ee79d8d10d995, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883197653 2024-11-17T22:39:58,993 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 482f0dde04084951bab14c9944876b78, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883197653 2024-11-17T22:39:58,993 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d60058b31c64ec195d5ca3dce6f3a04, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731883197715 2024-11-17T22:39:58,994 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cce1f43d4914273b4d2bc31f66d9a99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731883197715 2024-11-17T22:39:58,994 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c2747e3527d4c11b7c89c57ac7d2243, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731883198751 2024-11-17T22:39:58,995 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cf936d1dcb3401a8fa55d95196391fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731883198751 2024-11-17T22:39:59,012 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:39:59,013 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/ae42126497f1494f8793a6c8214affe8 is 50, key is test_row_0/A:col10/1731883198753/Put/seqid=0 2024-11-17T22:39:59,020 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:39:59,022 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/dd75aa14d8ff4f7c9840c9eeff25bf47 is 50, key is test_row_0/B:col10/1731883198753/Put/seqid=0 2024-11-17T22:39:59,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741855_1031 (size=12207) 2024-11-17T22:39:59,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741856_1032 (size=12207) 2024-11-17T22:39:59,074 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/ae42126497f1494f8793a6c8214affe8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/ae42126497f1494f8793a6c8214affe8 2024-11-17T22:39:59,088 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/dd75aa14d8ff4f7c9840c9eeff25bf47 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/dd75aa14d8ff4f7c9840c9eeff25bf47 2024-11-17T22:39:59,101 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/A of dde7707c5ff08c6cad672aef50110927 into ae42126497f1494f8793a6c8214affe8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:39:59,101 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:59,101 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/A, priority=13, startTime=1731883198986; duration=0sec 2024-11-17T22:39:59,101 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:39:59,102 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:A 2024-11-17T22:39:59,102 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:39:59,102 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/B of dde7707c5ff08c6cad672aef50110927 into dd75aa14d8ff4f7c9840c9eeff25bf47(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:39:59,103 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:59,103 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/B, priority=13, startTime=1731883198987; duration=0sec 2024-11-17T22:39:59,103 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:39:59,103 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:B 2024-11-17T22:39:59,106 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:39:59,106 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/C is initiating minor compaction (all files) 2024-11-17T22:39:59,106 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/C in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:59,107 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/214ee6f0fbcb421bb58a69727f21372e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/d6a3250dc0bc41a288f61f85063e78dd, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f56542f3159940918ea1e8c5ea4a77c0] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=35.3 K 2024-11-17T22:39:59,109 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 214ee6f0fbcb421bb58a69727f21372e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883197653 2024-11-17T22:39:59,110 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6a3250dc0bc41a288f61f85063e78dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731883197715 2024-11-17T22:39:59,111 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting f56542f3159940918ea1e8c5ea4a77c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731883198751 2024-11-17T22:39:59,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T22:39:59,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:59,125 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:39:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/af2c3b3c503c4b7f9117826baa5c7044 is 50, key is test_row_0/A:col10/1731883198821/Put/seqid=0 2024-11-17T22:39:59,139 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:39:59,140 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/34f3e6f386814222818069055369fe1a is 50, key is test_row_0/C:col10/1731883198753/Put/seqid=0 2024-11-17T22:39:59,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741857_1033 (size=12001) 2024-11-17T22:39:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:39:59,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883259189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883259192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883259195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883259197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741858_1034 (size=12207) 2024-11-17T22:39:59,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883259308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883259307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883259308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883259308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883259518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883259518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883259518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883259519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,569 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/af2c3b3c503c4b7f9117826baa5c7044 2024-11-17T22:39:59,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/869f5ec04eec471c8dc31bb1f28d947f is 50, key is test_row_0/B:col10/1731883198821/Put/seqid=0 2024-11-17T22:39:59,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741859_1035 (size=12001) 2024-11-17T22:39:59,617 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/869f5ec04eec471c8dc31bb1f28d947f 2024-11-17T22:39:59,632 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/34f3e6f386814222818069055369fe1a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/34f3e6f386814222818069055369fe1a 2024-11-17T22:39:59,647 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/C of dde7707c5ff08c6cad672aef50110927 into 34f3e6f386814222818069055369fe1a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:39:59,647 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:59,648 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/C, priority=13, startTime=1731883198987; duration=0sec 2024-11-17T22:39:59,648 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:39:59,648 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:C 2024-11-17T22:39:59,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/41a03858a43341c1adbf6eb9ebef551a is 50, key is test_row_0/C:col10/1731883198821/Put/seqid=0 2024-11-17T22:39:59,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741860_1036 (size=12001) 2024-11-17T22:39:59,681 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/41a03858a43341c1adbf6eb9ebef551a 2024-11-17T22:39:59,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/af2c3b3c503c4b7f9117826baa5c7044 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af2c3b3c503c4b7f9117826baa5c7044 2024-11-17T22:39:59,723 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af2c3b3c503c4b7f9117826baa5c7044, entries=150, sequenceid=119, filesize=11.7 K 2024-11-17T22:39:59,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/869f5ec04eec471c8dc31bb1f28d947f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/869f5ec04eec471c8dc31bb1f28d947f 2024-11-17T22:39:59,744 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/869f5ec04eec471c8dc31bb1f28d947f, entries=150, sequenceid=119, filesize=11.7 K 2024-11-17T22:39:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/41a03858a43341c1adbf6eb9ebef551a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/41a03858a43341c1adbf6eb9ebef551a 2024-11-17T22:39:59,763 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/41a03858a43341c1adbf6eb9ebef551a, entries=150, sequenceid=119, filesize=11.7 K 2024-11-17T22:39:59,765 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for dde7707c5ff08c6cad672aef50110927 in 641ms, sequenceid=119, compaction requested=false 2024-11-17T22:39:59,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:39:59,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:39:59,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-17T22:39:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-17T22:39:59,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-17T22:39:59,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0570 sec 2024-11-17T22:39:59,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.0810 sec 2024-11-17T22:39:59,831 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:39:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:39:59,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:39:59,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/d306b0bd5ea546859ec9a355f14976b1 is 50, key is test_row_0/A:col10/1731883199191/Put/seqid=0 2024-11-17T22:39:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T22:39:59,858 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:39:59,859 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:39:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-17T22:39:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T22:39:59,863 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:39:59,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741861_1037 (size=12101) 2024-11-17T22:39:59,864 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:39:59,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:39:59,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/d306b0bd5ea546859ec9a355f14976b1 2024-11-17T22:39:59,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/e3498b00009d4609b242b41b15410d18 is 50, key is test_row_0/B:col10/1731883199191/Put/seqid=0 2024-11-17T22:39:59,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883259886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883259890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883259895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883259897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741862_1038 (size=12101) 2024-11-17T22:39:59,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/e3498b00009d4609b242b41b15410d18 2024-11-17T22:39:59,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/0df011f8ba2a4173893bce9f1a953e61 is 50, key is test_row_0/C:col10/1731883199191/Put/seqid=0 2024-11-17T22:39:59,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:39:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883259947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:39:59,957 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in dde7707c5ff08c6cad672aef50110927 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:39:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:39:59,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:39:59,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741863_1039 (size=12101) 2024-11-17T22:39:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T22:39:59,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/0df011f8ba2a4173893bce9f1a953e61 2024-11-17T22:39:59,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/d306b0bd5ea546859ec9a355f14976b1 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d306b0bd5ea546859ec9a355f14976b1 2024-11-17T22:40:00,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d306b0bd5ea546859ec9a355f14976b1, entries=150, sequenceid=134, filesize=11.8 K 2024-11-17T22:40:00,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/e3498b00009d4609b242b41b15410d18 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e3498b00009d4609b242b41b15410d18 2024-11-17T22:40:00,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883260007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T22:40:00,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:00,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:00,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:00,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883260007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883260008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883260008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e3498b00009d4609b242b41b15410d18, entries=150, sequenceid=134, filesize=11.8 K 2024-11-17T22:40:00,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/0df011f8ba2a4173893bce9f1a953e61 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/0df011f8ba2a4173893bce9f1a953e61 2024-11-17T22:40:00,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/0df011f8ba2a4173893bce9f1a953e61, entries=150, sequenceid=134, filesize=11.8 K 2024-11-17T22:40:00,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for dde7707c5ff08c6cad672aef50110927 in 217ms, sequenceid=134, compaction requested=true 2024-11-17T22:40:00,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:00,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:00,049 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:00,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:00,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:00,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:00,050 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:00,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:00,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:00,052 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:00,052 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:00,052 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/B is initiating minor compaction (all files) 2024-11-17T22:40:00,052 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/A is initiating minor compaction (all files) 2024-11-17T22:40:00,052 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/A in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,052 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/B in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,053 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/ae42126497f1494f8793a6c8214affe8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af2c3b3c503c4b7f9117826baa5c7044, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d306b0bd5ea546859ec9a355f14976b1] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=35.5 K 2024-11-17T22:40:00,053 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/dd75aa14d8ff4f7c9840c9eeff25bf47, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/869f5ec04eec471c8dc31bb1f28d947f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e3498b00009d4609b242b41b15410d18] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=35.5 K 2024-11-17T22:40:00,053 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae42126497f1494f8793a6c8214affe8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731883198751 2024-11-17T22:40:00,053 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting dd75aa14d8ff4f7c9840c9eeff25bf47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731883198751 2024-11-17T22:40:00,054 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 869f5ec04eec471c8dc31bb1f28d947f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731883198821 2024-11-17T22:40:00,054 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting af2c3b3c503c4b7f9117826baa5c7044, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731883198821 2024-11-17T22:40:00,055 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting e3498b00009d4609b242b41b15410d18, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731883199174 2024-11-17T22:40:00,055 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting d306b0bd5ea546859ec9a355f14976b1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731883199174 2024-11-17T22:40:00,072 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:00,072 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:00,073 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/7ef5c4306f73489d96dfbe31d85708c6 is 50, key is test_row_0/A:col10/1731883199191/Put/seqid=0 2024-11-17T22:40:00,073 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/14b039f7ece44cc3b925800ca4f97233 is 50, key is test_row_0/B:col10/1731883199191/Put/seqid=0 2024-11-17T22:40:00,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741864_1040 (size=12409) 2024-11-17T22:40:00,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741865_1041 (size=12409) 2024-11-17T22:40:00,130 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/14b039f7ece44cc3b925800ca4f97233 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/14b039f7ece44cc3b925800ca4f97233 2024-11-17T22:40:00,146 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/7ef5c4306f73489d96dfbe31d85708c6 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7ef5c4306f73489d96dfbe31d85708c6 2024-11-17T22:40:00,146 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/B of dde7707c5ff08c6cad672aef50110927 into 14b039f7ece44cc3b925800ca4f97233(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:00,147 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:00,147 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/B, priority=13, startTime=1731883200049; duration=0sec 2024-11-17T22:40:00,147 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:00,147 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:B 2024-11-17T22:40:00,148 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:00,150 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:00,151 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/C is initiating minor compaction (all files) 2024-11-17T22:40:00,152 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/C in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,152 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/34f3e6f386814222818069055369fe1a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/41a03858a43341c1adbf6eb9ebef551a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/0df011f8ba2a4173893bce9f1a953e61] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=35.5 K 2024-11-17T22:40:00,153 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 34f3e6f386814222818069055369fe1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731883198751 2024-11-17T22:40:00,154 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a03858a43341c1adbf6eb9ebef551a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731883198821 2024-11-17T22:40:00,156 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df011f8ba2a4173893bce9f1a953e61, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731883199174 2024-11-17T22:40:00,160 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/A of dde7707c5ff08c6cad672aef50110927 into 7ef5c4306f73489d96dfbe31d85708c6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:00,160 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:00,161 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/A, priority=13, startTime=1731883200049; duration=0sec 2024-11-17T22:40:00,161 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:00,161 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:A 2024-11-17T22:40:00,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T22:40:00,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,175 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T22:40:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T22:40:00,187 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#C#compaction#29 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:00,188 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/22ed90edc1644f59824880be946ddd5a is 50, key is test_row_0/C:col10/1731883199191/Put/seqid=0 2024-11-17T22:40:00,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/c14a7f58d8334d89b2a83635a3e774d6 is 50, key is test_row_0/A:col10/1731883199857/Put/seqid=0 2024-11-17T22:40:00,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741866_1042 (size=12409) 2024-11-17T22:40:00,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741867_1043 (size=12151) 2024-11-17T22:40:00,226 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/c14a7f58d8334d89b2a83635a3e774d6 2024-11-17T22:40:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:00,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:00,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/626a86fe61b544bca47543e2b5486b5a is 50, key is test_row_0/B:col10/1731883199857/Put/seqid=0 2024-11-17T22:40:00,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883260256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883260258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883260266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741868_1044 (size=12151) 2024-11-17T22:40:00,276 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/626a86fe61b544bca47543e2b5486b5a 2024-11-17T22:40:00,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883260268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/665d6ab65b4f4ba49b329170dcd091f5 is 50, key is test_row_0/C:col10/1731883199857/Put/seqid=0 2024-11-17T22:40:00,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741869_1045 (size=12151) 2024-11-17T22:40:00,329 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/665d6ab65b4f4ba49b329170dcd091f5 2024-11-17T22:40:00,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/c14a7f58d8334d89b2a83635a3e774d6 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/c14a7f58d8334d89b2a83635a3e774d6 2024-11-17T22:40:00,355 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/c14a7f58d8334d89b2a83635a3e774d6, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T22:40:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/626a86fe61b544bca47543e2b5486b5a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/626a86fe61b544bca47543e2b5486b5a 2024-11-17T22:40:00,366 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/626a86fe61b544bca47543e2b5486b5a, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T22:40:00,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/665d6ab65b4f4ba49b329170dcd091f5 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/665d6ab65b4f4ba49b329170dcd091f5 2024-11-17T22:40:00,378 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/665d6ab65b4f4ba49b329170dcd091f5, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T22:40:00,417 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for dde7707c5ff08c6cad672aef50110927 in 242ms, sequenceid=158, compaction requested=false 2024-11-17T22:40:00,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:00,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-17T22:40:00,418 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:00,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-17T22:40:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-17T22:40:00,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/78d6c9b6dae9423c982f10d390da8f03 is 50, key is test_row_0/A:col10/1731883200260/Put/seqid=0 2024-11-17T22:40:00,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-17T22:40:00,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 558 msec 2024-11-17T22:40:00,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 577 msec 2024-11-17T22:40:00,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741870_1046 (size=14541) 2024-11-17T22:40:00,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883260473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883260479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T22:40:00,488 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:00,488 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883260482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883260482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-17T22:40:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T22:40:00,493 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:00,495 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:00,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:00,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883260588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883260588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T22:40:00,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883260599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883260601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,629 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/22ed90edc1644f59824880be946ddd5a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/22ed90edc1644f59824880be946ddd5a 2024-11-17T22:40:00,642 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/C of dde7707c5ff08c6cad672aef50110927 into 22ed90edc1644f59824880be946ddd5a(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:00,642 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:00,642 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/C, priority=13, startTime=1731883200050; duration=0sec 2024-11-17T22:40:00,642 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:00,642 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:C 2024-11-17T22:40:00,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T22:40:00,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:00,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:00,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:00,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883260798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883260798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T22:40:00,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:00,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:00,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T22:40:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:00,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883260807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883260831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:00,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/78d6c9b6dae9423c982f10d390da8f03 2024-11-17T22:40:00,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/484780c37f804de8a41791ee01d7cdb6 is 50, key is test_row_0/B:col10/1731883200260/Put/seqid=0 2024-11-17T22:40:00,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741871_1047 (size=12151) 2024-11-17T22:40:00,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T22:40:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:00,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:01,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883261107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883261107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T22:40:01,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T22:40:01,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:01,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:01,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:01,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:01,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883261117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883261147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T22:40:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:01,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:01,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:01,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/484780c37f804de8a41791ee01d7cdb6 2024-11-17T22:40:01,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/45b3c56ebb27481d836c7c9854638cac is 50, key is test_row_0/C:col10/1731883200260/Put/seqid=0 2024-11-17T22:40:01,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741872_1048 (size=12151) 2024-11-17T22:40:01,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/45b3c56ebb27481d836c7c9854638cac 2024-11-17T22:40:01,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/78d6c9b6dae9423c982f10d390da8f03 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/78d6c9b6dae9423c982f10d390da8f03 2024-11-17T22:40:01,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/78d6c9b6dae9423c982f10d390da8f03, entries=200, sequenceid=176, filesize=14.2 K 2024-11-17T22:40:01,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/484780c37f804de8a41791ee01d7cdb6 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/484780c37f804de8a41791ee01d7cdb6 2024-11-17T22:40:01,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/484780c37f804de8a41791ee01d7cdb6, entries=150, sequenceid=176, filesize=11.9 K 2024-11-17T22:40:01,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/45b3c56ebb27481d836c7c9854638cac as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/45b3c56ebb27481d836c7c9854638cac 2024-11-17T22:40:01,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/45b3c56ebb27481d836c7c9854638cac, entries=150, sequenceid=176, filesize=11.9 K 2024-11-17T22:40:01,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for dde7707c5ff08c6cad672aef50110927 in 973ms, sequenceid=176, compaction requested=true 2024-11-17T22:40:01,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:01,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:01,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:01,392 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:01,392 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:01,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:01,394 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:01,394 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/A is initiating minor compaction (all files) 2024-11-17T22:40:01,394 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/A in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:01,394 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7ef5c4306f73489d96dfbe31d85708c6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/c14a7f58d8334d89b2a83635a3e774d6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/78d6c9b6dae9423c982f10d390da8f03] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=38.2 K 2024-11-17T22:40:01,396 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:01,396 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/B is initiating minor compaction (all files) 2024-11-17T22:40:01,396 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/B in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:01,396 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/14b039f7ece44cc3b925800ca4f97233, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/626a86fe61b544bca47543e2b5486b5a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/484780c37f804de8a41791ee01d7cdb6] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=35.9 K 2024-11-17T22:40:01,396 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ef5c4306f73489d96dfbe31d85708c6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731883199174 2024-11-17T22:40:01,397 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 14b039f7ece44cc3b925800ca4f97233, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731883199174 2024-11-17T22:40:01,397 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting c14a7f58d8334d89b2a83635a3e774d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731883199857 2024-11-17T22:40:01,398 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 626a86fe61b544bca47543e2b5486b5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731883199857 2024-11-17T22:40:01,398 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78d6c9b6dae9423c982f10d390da8f03, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731883200260 2024-11-17T22:40:01,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:01,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:01,398 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 484780c37f804de8a41791ee01d7cdb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731883200260 2024-11-17T22:40:01,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:01,416 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:01,416 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#A#compaction#36 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:01,417 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/c2053f4c2a4d4f29a64edd40826b6216 is 50, key is test_row_0/B:col10/1731883200260/Put/seqid=0 2024-11-17T22:40:01,417 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/bde7855d25e14fc580858c7815a7357f is 50, key is test_row_0/A:col10/1731883200260/Put/seqid=0 2024-11-17T22:40:01,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T22:40:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:01,426 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T22:40:01,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741874_1050 (size=12561) 2024-11-17T22:40:01,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741873_1049 (size=12561) 2024-11-17T22:40:01,490 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/c2053f4c2a4d4f29a64edd40826b6216 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/c2053f4c2a4d4f29a64edd40826b6216 2024-11-17T22:40:01,492 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/bde7855d25e14fc580858c7815a7357f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/bde7855d25e14fc580858c7815a7357f 2024-11-17T22:40:01,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/3f499750a4894d4da3e7942bc2f5a299 is 50, key is test_row_0/A:col10/1731883200476/Put/seqid=0 2024-11-17T22:40:01,503 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/B of dde7707c5ff08c6cad672aef50110927 into c2053f4c2a4d4f29a64edd40826b6216(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:01,503 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:01,503 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/B, priority=13, startTime=1731883201392; duration=0sec 2024-11-17T22:40:01,504 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:01,504 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:B 2024-11-17T22:40:01,504 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:01,506 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/A of dde7707c5ff08c6cad672aef50110927 into bde7855d25e14fc580858c7815a7357f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:01,506 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:01,506 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/A, priority=13, startTime=1731883201392; duration=0sec 2024-11-17T22:40:01,507 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:01,507 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:A 2024-11-17T22:40:01,508 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:01,508 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/C is initiating minor compaction (all files) 2024-11-17T22:40:01,508 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/C in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:01,508 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/22ed90edc1644f59824880be946ddd5a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/665d6ab65b4f4ba49b329170dcd091f5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/45b3c56ebb27481d836c7c9854638cac] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=35.9 K 2024-11-17T22:40:01,509 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 22ed90edc1644f59824880be946ddd5a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731883199174 2024-11-17T22:40:01,510 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 665d6ab65b4f4ba49b329170dcd091f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731883199857 2024-11-17T22:40:01,512 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b3c56ebb27481d836c7c9854638cac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731883200260 2024-11-17T22:40:01,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741875_1051 (size=12151) 2024-11-17T22:40:01,531 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/3f499750a4894d4da3e7942bc2f5a299 2024-11-17T22:40:01,537 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:01,538 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/fc750c18784e46b190f6b7e8dd8628d9 is 50, key is test_row_0/C:col10/1731883200260/Put/seqid=0 2024-11-17T22:40:01,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/e0bee3ce4b624592bcce2f9f1570dfd4 is 50, key is test_row_0/B:col10/1731883200476/Put/seqid=0 2024-11-17T22:40:01,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741876_1052 (size=12561) 2024-11-17T22:40:01,603 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/fc750c18784e46b190f6b7e8dd8628d9 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/fc750c18784e46b190f6b7e8dd8628d9 2024-11-17T22:40:01,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741877_1053 (size=12151) 2024-11-17T22:40:01,612 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/e0bee3ce4b624592bcce2f9f1570dfd4 2024-11-17T22:40:01,622 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/C of dde7707c5ff08c6cad672aef50110927 into fc750c18784e46b190f6b7e8dd8628d9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:01,622 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:01,622 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/C, priority=13, startTime=1731883201398; duration=0sec 2024-11-17T22:40:01,623 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:01,623 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:C 2024-11-17T22:40:01,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T22:40:01,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/3d88b187d9094dd081f2e56ca9d4689f is 50, key is test_row_0/C:col10/1731883200476/Put/seqid=0 2024-11-17T22:40:01,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741878_1054 (size=12151) 2024-11-17T22:40:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883261671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883261669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883261677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883261678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883261787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883261788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883261787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883261977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,980 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in dde7707c5ff08c6cad672aef50110927 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:40:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883261997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883261997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883261998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,067 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/3d88b187d9094dd081f2e56ca9d4689f 2024-11-17T22:40:02,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/3f499750a4894d4da3e7942bc2f5a299 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3f499750a4894d4da3e7942bc2f5a299 2024-11-17T22:40:02,087 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3f499750a4894d4da3e7942bc2f5a299, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T22:40:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/e0bee3ce4b624592bcce2f9f1570dfd4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e0bee3ce4b624592bcce2f9f1570dfd4 2024-11-17T22:40:02,098 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e0bee3ce4b624592bcce2f9f1570dfd4, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T22:40:02,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/3d88b187d9094dd081f2e56ca9d4689f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3d88b187d9094dd081f2e56ca9d4689f 2024-11-17T22:40:02,110 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3d88b187d9094dd081f2e56ca9d4689f, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T22:40:02,111 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for dde7707c5ff08c6cad672aef50110927 in 685ms, sequenceid=195, compaction requested=false 2024-11-17T22:40:02,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:02,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:02,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-17T22:40:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-17T22:40:02,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-17T22:40:02,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6180 sec 2024-11-17T22:40:02,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6290 sec 2024-11-17T22:40:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:02,310 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:02,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T22:40:02,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/737311e649b74e4996b3f5ac951c40e1 is 50, key is test_row_0/A:col10/1731883202308/Put/seqid=0 2024-11-17T22:40:02,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741879_1055 (size=16931) 2024-11-17T22:40:02,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883262345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883262345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883262345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/737311e649b74e4996b3f5ac951c40e1 2024-11-17T22:40:02,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/936a0394602a4f21a1a212d3741fb503 is 50, key is test_row_0/B:col10/1731883202308/Put/seqid=0 2024-11-17T22:40:02,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741880_1056 (size=12151) 2024-11-17T22:40:02,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883262457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883262457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883262458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T22:40:02,648 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:02,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-17T22:40:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T22:40:02,656 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:02,657 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:02,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883262667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883262668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883262670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883262687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,690 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in dde7707c5ff08c6cad672aef50110927 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:40:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T22:40:02,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/936a0394602a4f21a1a212d3741fb503 2024-11-17T22:40:02,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/b8056e849a3545a28a3ba40d031810ce is 50, key is test_row_0/C:col10/1731883202308/Put/seqid=0 2024-11-17T22:40:02,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741881_1057 (size=12151) 2024-11-17T22:40:02,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T22:40:02,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:02,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:02,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:02,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:02,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:02,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T22:40:02,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:02,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:02,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:02,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:02,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T22:40:02,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883262978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883262978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883262979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:03,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T22:40:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:03,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:03,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:03,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:03,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/b8056e849a3545a28a3ba40d031810ce 2024-11-17T22:40:03,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/737311e649b74e4996b3f5ac951c40e1 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/737311e649b74e4996b3f5ac951c40e1 2024-11-17T22:40:03,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/737311e649b74e4996b3f5ac951c40e1, entries=250, sequenceid=218, filesize=16.5 K 2024-11-17T22:40:03,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/936a0394602a4f21a1a212d3741fb503 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/936a0394602a4f21a1a212d3741fb503 2024-11-17T22:40:03,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/936a0394602a4f21a1a212d3741fb503, entries=150, sequenceid=218, filesize=11.9 K 2024-11-17T22:40:03,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/b8056e849a3545a28a3ba40d031810ce as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b8056e849a3545a28a3ba40d031810ce 2024-11-17T22:40:03,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b8056e849a3545a28a3ba40d031810ce, entries=150, sequenceid=218, filesize=11.9 K 2024-11-17T22:40:03,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for dde7707c5ff08c6cad672aef50110927 in 923ms, sequenceid=218, compaction requested=true 2024-11-17T22:40:03,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:03,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:03,233 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:03,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:03,233 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:03,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:03,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:03,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:03,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:03,235 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:03,235 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/B is initiating minor compaction (all files) 2024-11-17T22:40:03,235 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/B in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:03,235 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:03,235 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/c2053f4c2a4d4f29a64edd40826b6216, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e0bee3ce4b624592bcce2f9f1570dfd4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/936a0394602a4f21a1a212d3741fb503] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=36.0 K 2024-11-17T22:40:03,235 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/A is initiating minor compaction (all files) 2024-11-17T22:40:03,235 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/A in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:03,236 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/bde7855d25e14fc580858c7815a7357f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3f499750a4894d4da3e7942bc2f5a299, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/737311e649b74e4996b3f5ac951c40e1] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=40.7 K 2024-11-17T22:40:03,236 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting c2053f4c2a4d4f29a64edd40826b6216, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731883200260 2024-11-17T22:40:03,236 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting bde7855d25e14fc580858c7815a7357f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731883200260 2024-11-17T22:40:03,237 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f499750a4894d4da3e7942bc2f5a299, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731883200476 2024-11-17T22:40:03,237 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting e0bee3ce4b624592bcce2f9f1570dfd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731883200476 2024-11-17T22:40:03,237 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 737311e649b74e4996b3f5ac951c40e1, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731883201658 2024-11-17T22:40:03,238 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 936a0394602a4f21a1a212d3741fb503, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731883201660 2024-11-17T22:40:03,253 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:03,254 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/8935387dc3bc4258a4563cff65bd145a is 50, key is test_row_0/B:col10/1731883202308/Put/seqid=0 2024-11-17T22:40:03,261 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:03,262 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/08a8fd2ea65d4de19103c2300215c1f2 is 50, key is test_row_0/A:col10/1731883202308/Put/seqid=0 2024-11-17T22:40:03,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T22:40:03,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:03,285 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T22:40:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T22:40:03,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/b3d6bf5f985e438bae2f6ce0fc5632f7 is 50, key is test_row_0/A:col10/1731883202342/Put/seqid=0 2024-11-17T22:40:03,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741882_1058 (size=12663) 2024-11-17T22:40:03,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741883_1059 (size=12663) 2024-11-17T22:40:03,310 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/08a8fd2ea65d4de19103c2300215c1f2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/08a8fd2ea65d4de19103c2300215c1f2 2024-11-17T22:40:03,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741884_1060 (size=12151) 2024-11-17T22:40:03,322 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/A of dde7707c5ff08c6cad672aef50110927 into 08a8fd2ea65d4de19103c2300215c1f2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:03,322 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:03,322 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/A, priority=13, startTime=1731883203233; duration=0sec 2024-11-17T22:40:03,322 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:03,322 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:A 2024-11-17T22:40:03,323 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:03,324 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:03,324 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/C is initiating minor compaction (all files) 2024-11-17T22:40:03,325 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/C in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:03,325 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/fc750c18784e46b190f6b7e8dd8628d9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3d88b187d9094dd081f2e56ca9d4689f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b8056e849a3545a28a3ba40d031810ce] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=36.0 K 2024-11-17T22:40:03,325 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc750c18784e46b190f6b7e8dd8628d9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731883200260 2024-11-17T22:40:03,326 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d88b187d9094dd081f2e56ca9d4689f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731883200476 2024-11-17T22:40:03,326 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8056e849a3545a28a3ba40d031810ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731883201660 2024-11-17T22:40:03,339 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:03,340 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/0e1f7eb1b5b8430c94d2136e6f944107 is 50, key is test_row_0/C:col10/1731883202308/Put/seqid=0 2024-11-17T22:40:03,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741885_1061 (size=12663) 2024-11-17T22:40:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:03,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:03,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883263524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:03,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883263525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:03,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883263527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:03,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:03,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883263637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:03,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883263637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883263637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:03,703 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/8935387dc3bc4258a4563cff65bd145a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8935387dc3bc4258a4563cff65bd145a 2024-11-17T22:40:03,710 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/B of dde7707c5ff08c6cad672aef50110927 into 8935387dc3bc4258a4563cff65bd145a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:03,710 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:03,711 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/B, priority=13, startTime=1731883203233; duration=0sec 2024-11-17T22:40:03,711 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:03,711 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:B 2024-11-17T22:40:03,716 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/b3d6bf5f985e438bae2f6ce0fc5632f7 2024-11-17T22:40:03,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/3e3c5ebbb84349078ba5417496d2d122 is 50, key is test_row_0/B:col10/1731883202342/Put/seqid=0 2024-11-17T22:40:03,755 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/0e1f7eb1b5b8430c94d2136e6f944107 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/0e1f7eb1b5b8430c94d2136e6f944107 2024-11-17T22:40:03,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741886_1062 (size=12151) 2024-11-17T22:40:03,765 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/3e3c5ebbb84349078ba5417496d2d122 2024-11-17T22:40:03,778 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/C of dde7707c5ff08c6cad672aef50110927 into 0e1f7eb1b5b8430c94d2136e6f944107(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:03,778 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:03,778 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/C, priority=13, startTime=1731883203234; duration=0sec 2024-11-17T22:40:03,778 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:03,778 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:C 2024-11-17T22:40:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/3a47e5c9ebee4b89bd6763b1addfaf81 is 50, key is test_row_0/C:col10/1731883202342/Put/seqid=0 2024-11-17T22:40:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T22:40:03,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741887_1063 (size=12151) 2024-11-17T22:40:03,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883263847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:03,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883263848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:03,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883263848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883264157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883264157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:04,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883264158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,212 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/3a47e5c9ebee4b89bd6763b1addfaf81 2024-11-17T22:40:04,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/b3d6bf5f985e438bae2f6ce0fc5632f7 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b3d6bf5f985e438bae2f6ce0fc5632f7 2024-11-17T22:40:04,235 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b3d6bf5f985e438bae2f6ce0fc5632f7, entries=150, sequenceid=234, filesize=11.9 K 2024-11-17T22:40:04,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/3e3c5ebbb84349078ba5417496d2d122 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/3e3c5ebbb84349078ba5417496d2d122 2024-11-17T22:40:04,247 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/3e3c5ebbb84349078ba5417496d2d122, entries=150, sequenceid=234, filesize=11.9 K 2024-11-17T22:40:04,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/3a47e5c9ebee4b89bd6763b1addfaf81 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3a47e5c9ebee4b89bd6763b1addfaf81 2024-11-17T22:40:04,272 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3a47e5c9ebee4b89bd6763b1addfaf81, entries=150, sequenceid=234, filesize=11.9 K 2024-11-17T22:40:04,274 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for dde7707c5ff08c6cad672aef50110927 in 989ms, sequenceid=234, compaction requested=false 2024-11-17T22:40:04,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:04,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:04,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-17T22:40:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-17T22:40:04,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-17T22:40:04,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6190 sec 2024-11-17T22:40:04,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6330 sec 2024-11-17T22:40:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:04,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:04,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T22:40:04,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/b63166813ab244bab93afb0c5ed76a2a is 50, key is test_row_0/A:col10/1731883203506/Put/seqid=0 2024-11-17T22:40:04,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883264706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883264713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:04,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883264717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883264730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741888_1064 (size=17031) 2024-11-17T22:40:04,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/b63166813ab244bab93afb0c5ed76a2a 2024-11-17T22:40:04,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/00743dccc9a34b48b040a29ae4828151 is 50, key is test_row_0/B:col10/1731883203506/Put/seqid=0 2024-11-17T22:40:04,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741889_1065 (size=12201) 2024-11-17T22:40:04,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/00743dccc9a34b48b040a29ae4828151 2024-11-17T22:40:04,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/16a33d44d63a442c8378738216eedeca is 50, key is test_row_0/C:col10/1731883203506/Put/seqid=0 2024-11-17T22:40:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T22:40:04,808 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:04,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:04,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-17T22:40:04,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T22:40:04,818 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:04,822 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:04,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883264828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883264838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883264838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883264838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:04,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741890_1066 (size=12201) 2024-11-17T22:40:04,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/16a33d44d63a442c8378738216eedeca 2024-11-17T22:40:04,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/b63166813ab244bab93afb0c5ed76a2a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b63166813ab244bab93afb0c5ed76a2a 2024-11-17T22:40:04,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b63166813ab244bab93afb0c5ed76a2a, entries=250, sequenceid=259, filesize=16.6 K 2024-11-17T22:40:04,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/00743dccc9a34b48b040a29ae4828151 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/00743dccc9a34b48b040a29ae4828151 2024-11-17T22:40:04,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/00743dccc9a34b48b040a29ae4828151, entries=150, sequenceid=259, filesize=11.9 K 2024-11-17T22:40:04,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/16a33d44d63a442c8378738216eedeca as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/16a33d44d63a442c8378738216eedeca 2024-11-17T22:40:04,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/16a33d44d63a442c8378738216eedeca, entries=150, sequenceid=259, filesize=11.9 K 2024-11-17T22:40:04,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for dde7707c5ff08c6cad672aef50110927 in 224ms, sequenceid=259, compaction requested=true 2024-11-17T22:40:04,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:04,902 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:04,904 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41845 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:04,904 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/A is initiating minor compaction (all files) 2024-11-17T22:40:04,904 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/A in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:04,905 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/08a8fd2ea65d4de19103c2300215c1f2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b3d6bf5f985e438bae2f6ce0fc5632f7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b63166813ab244bab93afb0c5ed76a2a] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=40.9 K 2024-11-17T22:40:04,905 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08a8fd2ea65d4de19103c2300215c1f2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731883201660 2024-11-17T22:40:04,906 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3d6bf5f985e438bae2f6ce0fc5632f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731883202317 2024-11-17T22:40:04,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:04,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:04,907 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:04,907 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting b63166813ab244bab93afb0c5ed76a2a, keycount=250, bloomtype=ROW, size=16.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731883203506 2024-11-17T22:40:04,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:04,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:04,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:04,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:04,909 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:04,909 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/B is initiating minor compaction (all files) 2024-11-17T22:40:04,909 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/B in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:04,910 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8935387dc3bc4258a4563cff65bd145a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/3e3c5ebbb84349078ba5417496d2d122, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/00743dccc9a34b48b040a29ae4828151] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=36.1 K 2024-11-17T22:40:04,911 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 8935387dc3bc4258a4563cff65bd145a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731883201660 2024-11-17T22:40:04,912 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e3c5ebbb84349078ba5417496d2d122, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731883202317 2024-11-17T22:40:04,913 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 00743dccc9a34b48b040a29ae4828151, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731883203506 2024-11-17T22:40:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T22:40:04,932 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#A#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:04,933 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/e1b9105fd0854c1e979b62bed40db04e is 50, key is test_row_0/A:col10/1731883203506/Put/seqid=0 2024-11-17T22:40:04,955 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#B#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:04,956 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/cdf3aec9b9644d9daba77c470e2067e2 is 50, key is test_row_0/B:col10/1731883203506/Put/seqid=0 2024-11-17T22:40:04,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T22:40:04,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:04,984 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T22:40:04,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741891_1067 (size=12815) 2024-11-17T22:40:04,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741892_1068 (size=12815) 2024-11-17T22:40:05,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/5d7cab0c049d4046a1d0654a0dc043a2 is 50, key is test_row_0/A:col10/1731883204682/Put/seqid=0 2024-11-17T22:40:05,013 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/cdf3aec9b9644d9daba77c470e2067e2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/cdf3aec9b9644d9daba77c470e2067e2 2024-11-17T22:40:05,037 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/B of dde7707c5ff08c6cad672aef50110927 into cdf3aec9b9644d9daba77c470e2067e2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:05,037 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:05,037 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/B, priority=13, startTime=1731883204907; duration=0sec 2024-11-17T22:40:05,037 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:05,037 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:B 2024-11-17T22:40:05,037 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:05,039 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:05,039 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/C is initiating minor compaction (all files) 2024-11-17T22:40:05,039 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/C in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:05,040 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/0e1f7eb1b5b8430c94d2136e6f944107, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3a47e5c9ebee4b89bd6763b1addfaf81, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/16a33d44d63a442c8378738216eedeca] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=36.1 K 2024-11-17T22:40:05,041 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e1f7eb1b5b8430c94d2136e6f944107, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731883201660 2024-11-17T22:40:05,042 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a47e5c9ebee4b89bd6763b1addfaf81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731883202317 2024-11-17T22:40:05,043 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 16a33d44d63a442c8378738216eedeca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731883203506 2024-11-17T22:40:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:05,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:05,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741893_1069 (size=12301) 2024-11-17T22:40:05,058 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/5d7cab0c049d4046a1d0654a0dc043a2 2024-11-17T22:40:05,078 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:05,079 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/eaba7d174ada4d23b3a41c0c5feac042 is 50, key is test_row_0/C:col10/1731883203506/Put/seqid=0 2024-11-17T22:40:05,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/4bdb261d4b484a7cb4e18997cf5cff86 is 50, key is test_row_0/B:col10/1731883204682/Put/seqid=0 2024-11-17T22:40:05,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741894_1070 (size=12815) 2024-11-17T22:40:05,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741895_1071 (size=12301) 2024-11-17T22:40:05,114 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/eaba7d174ada4d23b3a41c0c5feac042 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/eaba7d174ada4d23b3a41c0c5feac042 2024-11-17T22:40:05,118 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/4bdb261d4b484a7cb4e18997cf5cff86 2024-11-17T22:40:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T22:40:05,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/a4af620f37dd4a939d7371f7338de191 is 50, key is test_row_0/C:col10/1731883204682/Put/seqid=0 2024-11-17T22:40:05,145 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/C of dde7707c5ff08c6cad672aef50110927 into eaba7d174ada4d23b3a41c0c5feac042(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:05,146 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:05,146 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/C, priority=13, startTime=1731883204907; duration=0sec 2024-11-17T22:40:05,146 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:05,146 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:C 2024-11-17T22:40:05,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883265134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883265135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883265135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883265151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741896_1072 (size=12301) 2024-11-17T22:40:05,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883265258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883265258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883265258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883265277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,402 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/e1b9105fd0854c1e979b62bed40db04e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/e1b9105fd0854c1e979b62bed40db04e 2024-11-17T22:40:05,417 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/A of dde7707c5ff08c6cad672aef50110927 into e1b9105fd0854c1e979b62bed40db04e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:05,417 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:05,417 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/A, priority=13, startTime=1731883204902; duration=0sec 2024-11-17T22:40:05,418 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:05,418 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:A 2024-11-17T22:40:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T22:40:05,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883265467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883265476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883265477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883265487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,577 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/a4af620f37dd4a939d7371f7338de191 2024-11-17T22:40:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/5d7cab0c049d4046a1d0654a0dc043a2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/5d7cab0c049d4046a1d0654a0dc043a2 2024-11-17T22:40:05,606 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/5d7cab0c049d4046a1d0654a0dc043a2, entries=150, sequenceid=274, filesize=12.0 K 2024-11-17T22:40:05,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/4bdb261d4b484a7cb4e18997cf5cff86 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/4bdb261d4b484a7cb4e18997cf5cff86 2024-11-17T22:40:05,615 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/4bdb261d4b484a7cb4e18997cf5cff86, entries=150, sequenceid=274, filesize=12.0 K 2024-11-17T22:40:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/a4af620f37dd4a939d7371f7338de191 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/a4af620f37dd4a939d7371f7338de191 2024-11-17T22:40:05,638 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/a4af620f37dd4a939d7371f7338de191, entries=150, sequenceid=274, filesize=12.0 K 2024-11-17T22:40:05,639 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for dde7707c5ff08c6cad672aef50110927 in 656ms, sequenceid=274, compaction requested=false 2024-11-17T22:40:05,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:05,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:05,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-17T22:40:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-17T22:40:05,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-17T22:40:05,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 820 msec 2024-11-17T22:40:05,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 839 msec 2024-11-17T22:40:05,783 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:05,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T22:40:05,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/7857546d7a7a4c738978a92ada163c25 is 50, key is test_row_0/A:col10/1731883205781/Put/seqid=0 2024-11-17T22:40:05,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883265815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883265818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883265823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883265820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741897_1073 (size=14741) 2024-11-17T22:40:05,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/7857546d7a7a4c738978a92ada163c25 2024-11-17T22:40:05,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/0a582bc15fda42f198c8d13763410b6b is 50, key is test_row_0/B:col10/1731883205781/Put/seqid=0 2024-11-17T22:40:05,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741898_1074 (size=12301) 2024-11-17T22:40:05,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/0a582bc15fda42f198c8d13763410b6b 2024-11-17T22:40:05,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/1dd66a17399a402683373a3be3773a40 is 50, key is test_row_0/C:col10/1731883205781/Put/seqid=0 2024-11-17T22:40:05,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741899_1075 (size=12301) 2024-11-17T22:40:05,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883265937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883265937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883265937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883265937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T22:40:05,958 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:05,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-17T22:40:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T22:40:05,963 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:05,964 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:05,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:06,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44990 deadline: 1731883265997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,001 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in dde7707c5ff08c6cad672aef50110927 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:40:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T22:40:06,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T22:40:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:06,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:06,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883266147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883266147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883266147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883266157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T22:40:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:06,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T22:40:06,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/1dd66a17399a402683373a3be3773a40 2024-11-17T22:40:06,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/7857546d7a7a4c738978a92ada163c25 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7857546d7a7a4c738978a92ada163c25 2024-11-17T22:40:06,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7857546d7a7a4c738978a92ada163c25, entries=200, sequenceid=299, filesize=14.4 K 2024-11-17T22:40:06,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/0a582bc15fda42f198c8d13763410b6b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/0a582bc15fda42f198c8d13763410b6b 2024-11-17T22:40:06,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/0a582bc15fda42f198c8d13763410b6b, entries=150, sequenceid=299, filesize=12.0 K 2024-11-17T22:40:06,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/1dd66a17399a402683373a3be3773a40 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/1dd66a17399a402683373a3be3773a40 2024-11-17T22:40:06,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/1dd66a17399a402683373a3be3773a40, entries=150, sequenceid=299, filesize=12.0 K 2024-11-17T22:40:06,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for dde7707c5ff08c6cad672aef50110927 in 578ms, sequenceid=299, compaction requested=true 2024-11-17T22:40:06,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:06,361 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:06,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:06,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:06,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:06,363 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:06,363 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:06,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:06,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:06,363 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/A is initiating minor compaction (all files) 2024-11-17T22:40:06,363 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/A in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:06,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:06,363 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/e1b9105fd0854c1e979b62bed40db04e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/5d7cab0c049d4046a1d0654a0dc043a2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7857546d7a7a4c738978a92ada163c25] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=38.9 K 2024-11-17T22:40:06,364 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1b9105fd0854c1e979b62bed40db04e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731883203506 2024-11-17T22:40:06,364 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:06,364 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/B is initiating minor compaction (all files) 2024-11-17T22:40:06,364 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/B in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:06,364 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/cdf3aec9b9644d9daba77c470e2067e2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/4bdb261d4b484a7cb4e18997cf5cff86, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/0a582bc15fda42f198c8d13763410b6b] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=36.5 K 2024-11-17T22:40:06,365 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d7cab0c049d4046a1d0654a0dc043a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731883204682 2024-11-17T22:40:06,365 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting cdf3aec9b9644d9daba77c470e2067e2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731883203506 2024-11-17T22:40:06,367 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bdb261d4b484a7cb4e18997cf5cff86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731883204682 2024-11-17T22:40:06,367 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7857546d7a7a4c738978a92ada163c25, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731883205087 2024-11-17T22:40:06,368 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a582bc15fda42f198c8d13763410b6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731883205087 2024-11-17T22:40:06,399 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#A#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:06,400 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/4bab787366664ec3b18c9533a3edd226 is 50, key is test_row_0/A:col10/1731883205781/Put/seqid=0 2024-11-17T22:40:06,410 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:06,411 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/d688fd1329524d36825b2d3b67fec487 is 50, key is test_row_0/B:col10/1731883205781/Put/seqid=0 2024-11-17T22:40:06,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741900_1076 (size=13017) 2024-11-17T22:40:06,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T22:40:06,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:06,431 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T22:40:06,444 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/4bab787366664ec3b18c9533a3edd226 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/4bab787366664ec3b18c9533a3edd226 2024-11-17T22:40:06,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741901_1077 (size=13017) 2024-11-17T22:40:06,453 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/A of dde7707c5ff08c6cad672aef50110927 into 4bab787366664ec3b18c9533a3edd226(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:06,453 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:06,454 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/A, priority=13, startTime=1731883206361; duration=0sec 2024-11-17T22:40:06,454 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:06,454 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:A 2024-11-17T22:40:06,454 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:06,459 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:06,459 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/C is initiating minor compaction (all files) 2024-11-17T22:40:06,459 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/C in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:06,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/a2e296caffb646e1bba5c0e926a0eb71 is 50, key is test_row_0/A:col10/1731883205818/Put/seqid=0 2024-11-17T22:40:06,462 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/eaba7d174ada4d23b3a41c0c5feac042, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/a4af620f37dd4a939d7371f7338de191, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/1dd66a17399a402683373a3be3773a40] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=36.5 K 2024-11-17T22:40:06,463 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/d688fd1329524d36825b2d3b67fec487 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/d688fd1329524d36825b2d3b67fec487 2024-11-17T22:40:06,463 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaba7d174ada4d23b3a41c0c5feac042, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731883203506 2024-11-17T22:40:06,464 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4af620f37dd4a939d7371f7338de191, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731883204682 2024-11-17T22:40:06,465 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dd66a17399a402683373a3be3773a40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731883205087 2024-11-17T22:40:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:06,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:06,473 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/B of dde7707c5ff08c6cad672aef50110927 into d688fd1329524d36825b2d3b67fec487(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:06,473 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:06,473 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/B, priority=13, startTime=1731883206363; duration=0sec 2024-11-17T22:40:06,473 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:06,473 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:B 2024-11-17T22:40:06,495 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#C#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:06,496 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/235fd52f323448b182070533420741ef is 50, key is test_row_0/C:col10/1731883205781/Put/seqid=0 2024-11-17T22:40:06,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883266508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741902_1078 (size=12301) 2024-11-17T22:40:06,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883266509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,517 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/a2e296caffb646e1bba5c0e926a0eb71 2024-11-17T22:40:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883266512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883266514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/db323a99c3d34db3ace787d9e03d78a4 is 50, key is test_row_0/B:col10/1731883205818/Put/seqid=0 2024-11-17T22:40:06,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741903_1079 (size=13017) 2024-11-17T22:40:06,581 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/235fd52f323448b182070533420741ef as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/235fd52f323448b182070533420741ef 2024-11-17T22:40:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T22:40:06,595 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/C of dde7707c5ff08c6cad672aef50110927 into 235fd52f323448b182070533420741ef(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:06,595 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:06,595 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/C, priority=13, startTime=1731883206363; duration=0sec 2024-11-17T22:40:06,595 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:06,596 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:C 2024-11-17T22:40:06,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741904_1080 (size=12301) 2024-11-17T22:40:06,605 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/db323a99c3d34db3ace787d9e03d78a4 2024-11-17T22:40:06,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883266617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883266627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883266627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883266627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/63f38a18ff4741fa82586c34cfe5743a is 50, key is test_row_0/C:col10/1731883205818/Put/seqid=0 2024-11-17T22:40:06,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741905_1081 (size=12301) 2024-11-17T22:40:06,647 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/63f38a18ff4741fa82586c34cfe5743a 2024-11-17T22:40:06,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/a2e296caffb646e1bba5c0e926a0eb71 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/a2e296caffb646e1bba5c0e926a0eb71 2024-11-17T22:40:06,671 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/a2e296caffb646e1bba5c0e926a0eb71, entries=150, sequenceid=313, filesize=12.0 K 2024-11-17T22:40:06,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/db323a99c3d34db3ace787d9e03d78a4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/db323a99c3d34db3ace787d9e03d78a4 2024-11-17T22:40:06,683 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/db323a99c3d34db3ace787d9e03d78a4, entries=150, sequenceid=313, filesize=12.0 K 2024-11-17T22:40:06,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/63f38a18ff4741fa82586c34cfe5743a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/63f38a18ff4741fa82586c34cfe5743a 2024-11-17T22:40:06,694 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/63f38a18ff4741fa82586c34cfe5743a, entries=150, sequenceid=313, filesize=12.0 K 2024-11-17T22:40:06,696 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for dde7707c5ff08c6cad672aef50110927 in 265ms, sequenceid=313, compaction requested=false 2024-11-17T22:40:06,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:06,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:06,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-17T22:40:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-17T22:40:06,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-17T22:40:06,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 735 msec 2024-11-17T22:40:06,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 745 msec 2024-11-17T22:40:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:06,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:06,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T22:40:06,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/89c098bb551040a0afa538623836791a is 50, key is test_row_0/A:col10/1731883206511/Put/seqid=0 2024-11-17T22:40:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883266859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883266865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883266865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883266867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741906_1082 (size=14741) 2024-11-17T22:40:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883266977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883266977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883266977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883266977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T22:40:07,097 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:07,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-17T22:40:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T22:40:07,101 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:07,102 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:07,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:07,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883267187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883267187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:07,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883267187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:07,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883267187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T22:40:07,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T22:40:07,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:07,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/89c098bb551040a0afa538623836791a 2024-11-17T22:40:07,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/aa191f10ce5e4274aab1e70cef635373 is 50, key is test_row_0/B:col10/1731883206511/Put/seqid=0 2024-11-17T22:40:07,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741907_1083 (size=12301) 2024-11-17T22:40:07,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/aa191f10ce5e4274aab1e70cef635373 2024-11-17T22:40:07,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/f2abced56d384340b5928b48c4217c67 is 50, key is test_row_0/C:col10/1731883206511/Put/seqid=0 2024-11-17T22:40:07,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741908_1084 (size=12301) 2024-11-17T22:40:07,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T22:40:07,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:07,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T22:40:07,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883267497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:07,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883267497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:07,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:07,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883267497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883267497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:07,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T22:40:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T22:40:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T22:40:07,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/f2abced56d384340b5928b48c4217c67 2024-11-17T22:40:07,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/89c098bb551040a0afa538623836791a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/89c098bb551040a0afa538623836791a 2024-11-17T22:40:07,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/89c098bb551040a0afa538623836791a, entries=200, sequenceid=339, filesize=14.4 K 2024-11-17T22:40:07,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/aa191f10ce5e4274aab1e70cef635373 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/aa191f10ce5e4274aab1e70cef635373 2024-11-17T22:40:07,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/aa191f10ce5e4274aab1e70cef635373, entries=150, sequenceid=339, filesize=12.0 K 2024-11-17T22:40:07,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/f2abced56d384340b5928b48c4217c67 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f2abced56d384340b5928b48c4217c67 2024-11-17T22:40:07,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f2abced56d384340b5928b48c4217c67, entries=150, sequenceid=339, filesize=12.0 K 2024-11-17T22:40:07,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for dde7707c5ff08c6cad672aef50110927 in 951ms, sequenceid=339, compaction requested=true 2024-11-17T22:40:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:07,782 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:07,782 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:07,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:07,783 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:07,784 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/B is initiating minor compaction (all files) 2024-11-17T22:40:07,784 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/B in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,784 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/d688fd1329524d36825b2d3b67fec487, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/db323a99c3d34db3ace787d9e03d78a4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/aa191f10ce5e4274aab1e70cef635373] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=36.7 K 2024-11-17T22:40:07,785 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d688fd1329524d36825b2d3b67fec487, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731883205087 2024-11-17T22:40:07,787 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:07,787 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/A is initiating minor compaction (all files) 2024-11-17T22:40:07,787 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/A in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,787 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/4bab787366664ec3b18c9533a3edd226, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/a2e296caffb646e1bba5c0e926a0eb71, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/89c098bb551040a0afa538623836791a] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=39.1 K 2024-11-17T22:40:07,788 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting db323a99c3d34db3ace787d9e03d78a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731883205816 2024-11-17T22:40:07,788 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bab787366664ec3b18c9533a3edd226, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731883205087 2024-11-17T22:40:07,788 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting aa191f10ce5e4274aab1e70cef635373, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731883206497 2024-11-17T22:40:07,789 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2e296caffb646e1bba5c0e926a0eb71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731883205816 2024-11-17T22:40:07,790 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c098bb551040a0afa538623836791a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731883206497 2024-11-17T22:40:07,817 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:07,818 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/9ed7805f0eff419c95465541b185272e is 50, key is test_row_0/A:col10/1731883206511/Put/seqid=0 2024-11-17T22:40:07,831 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:07,832 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/97f2d4d3d4b44cb1aad3a136d0e7eb9b is 50, key is test_row_0/B:col10/1731883206511/Put/seqid=0 2024-11-17T22:40:07,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741909_1085 (size=13119) 2024-11-17T22:40:07,848 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/9ed7805f0eff419c95465541b185272e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/9ed7805f0eff419c95465541b185272e 2024-11-17T22:40:07,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741910_1086 (size=13119) 2024-11-17T22:40:07,862 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/A of dde7707c5ff08c6cad672aef50110927 into 9ed7805f0eff419c95465541b185272e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:07,862 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/97f2d4d3d4b44cb1aad3a136d0e7eb9b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/97f2d4d3d4b44cb1aad3a136d0e7eb9b 2024-11-17T22:40:07,862 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:07,863 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/A, priority=13, startTime=1731883207782; duration=0sec 2024-11-17T22:40:07,863 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:07,863 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:A 2024-11-17T22:40:07,863 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:07,865 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:07,865 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/C is initiating minor compaction (all files) 2024-11-17T22:40:07,865 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/C in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,865 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/235fd52f323448b182070533420741ef, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/63f38a18ff4741fa82586c34cfe5743a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f2abced56d384340b5928b48c4217c67] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=36.7 K 2024-11-17T22:40:07,866 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 235fd52f323448b182070533420741ef, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731883205087 2024-11-17T22:40:07,866 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63f38a18ff4741fa82586c34cfe5743a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731883205816 2024-11-17T22:40:07,867 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2abced56d384340b5928b48c4217c67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731883206497 2024-11-17T22:40:07,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T22:40:07,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:07,872 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:40:07,881 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/B of dde7707c5ff08c6cad672aef50110927 into 97f2d4d3d4b44cb1aad3a136d0e7eb9b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:07,881 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:07,881 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/B, priority=13, startTime=1731883207782; duration=0sec 2024-11-17T22:40:07,882 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:07,882 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:B 2024-11-17T22:40:07,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/66453494ee554c3c93ec53486f9a6506 is 50, key is test_row_0/A:col10/1731883206852/Put/seqid=0 2024-11-17T22:40:07,899 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#C#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:07,900 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/ddcccd771edb4e5eadc1565a2596b95b is 50, key is test_row_0/C:col10/1731883206511/Put/seqid=0 2024-11-17T22:40:07,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741911_1087 (size=12301) 2024-11-17T22:40:07,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741912_1088 (size=13119) 2024-11-17T22:40:07,951 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/ddcccd771edb4e5eadc1565a2596b95b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/ddcccd771edb4e5eadc1565a2596b95b 2024-11-17T22:40:07,962 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/C of dde7707c5ff08c6cad672aef50110927 into ddcccd771edb4e5eadc1565a2596b95b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:07,962 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:07,962 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/C, priority=13, startTime=1731883207782; duration=0sec 2024-11-17T22:40:07,963 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:07,963 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:C 2024-11-17T22:40:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:08,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:08,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883268048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883268053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883268057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883268058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883268166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883268167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883268167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883268177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T22:40:08,332 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/66453494ee554c3c93ec53486f9a6506 2024-11-17T22:40:08,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/ac5d53404ade4697a2f6e1ab5d70ee7d is 50, key is test_row_0/B:col10/1731883206852/Put/seqid=0 2024-11-17T22:40:08,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741913_1089 (size=12301) 2024-11-17T22:40:08,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883268376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883268377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883268377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883268387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883268687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883268687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883268687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883268697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:08,756 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/ac5d53404ade4697a2f6e1ab5d70ee7d 2024-11-17T22:40:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/75e47df4d42c44ac95dceadcf33eb178 is 50, key is test_row_0/C:col10/1731883206852/Put/seqid=0 2024-11-17T22:40:08,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741914_1090 (size=12301) 2024-11-17T22:40:09,190 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/75e47df4d42c44ac95dceadcf33eb178 2024-11-17T22:40:09,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/66453494ee554c3c93ec53486f9a6506 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/66453494ee554c3c93ec53486f9a6506 2024-11-17T22:40:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883269199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:09,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883269199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:09,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883269199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:09,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:09,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:09,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:09,208 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/66453494ee554c3c93ec53486f9a6506, entries=150, sequenceid=353, filesize=12.0 K 2024-11-17T22:40:09,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/ac5d53404ade4697a2f6e1ab5d70ee7d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ac5d53404ade4697a2f6e1ab5d70ee7d 2024-11-17T22:40:09,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883269209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:09,219 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ac5d53404ade4697a2f6e1ab5d70ee7d, entries=150, sequenceid=353, filesize=12.0 K 2024-11-17T22:40:09,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/75e47df4d42c44ac95dceadcf33eb178 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/75e47df4d42c44ac95dceadcf33eb178 2024-11-17T22:40:09,225 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/75e47df4d42c44ac95dceadcf33eb178, entries=150, sequenceid=353, filesize=12.0 K 2024-11-17T22:40:09,226 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for dde7707c5ff08c6cad672aef50110927 in 1354ms, sequenceid=353, compaction requested=false 2024-11-17T22:40:09,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:09,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:09,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-17T22:40:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-17T22:40:09,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-17T22:40:09,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1260 sec 2024-11-17T22:40:09,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1340 sec 2024-11-17T22:40:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T22:40:09,257 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:09,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-17T22:40:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T22:40:09,261 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:09,262 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:09,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T22:40:09,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T22:40:09,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:09,416 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:40:09,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/d10878a24ff14449a97ece1a09ab5373 is 50, key is test_row_0/A:col10/1731883208044/Put/seqid=0 2024-11-17T22:40:09,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741915_1091 (size=12301) 2024-11-17T22:40:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T22:40:09,851 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/d10878a24ff14449a97ece1a09ab5373 2024-11-17T22:40:09,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/56c685e87f5849568d9b26a3d65942a8 is 50, key is test_row_0/B:col10/1731883208044/Put/seqid=0 2024-11-17T22:40:09,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741916_1092 (size=12301) 2024-11-17T22:40:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T22:40:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:10,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:10,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883270221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883270226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883270228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883270230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,272 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/56c685e87f5849568d9b26a3d65942a8 2024-11-17T22:40:10,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/f69f6ec07d80410ba883098c99c51989 is 50, key is test_row_0/C:col10/1731883208044/Put/seqid=0 2024-11-17T22:40:10,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741917_1093 (size=12301) 2024-11-17T22:40:10,314 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/f69f6ec07d80410ba883098c99c51989 2024-11-17T22:40:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/d10878a24ff14449a97ece1a09ab5373 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d10878a24ff14449a97ece1a09ab5373 2024-11-17T22:40:10,330 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d10878a24ff14449a97ece1a09ab5373, entries=150, sequenceid=379, filesize=12.0 K 2024-11-17T22:40:10,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/56c685e87f5849568d9b26a3d65942a8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/56c685e87f5849568d9b26a3d65942a8 2024-11-17T22:40:10,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883270337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883270339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,343 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/56c685e87f5849568d9b26a3d65942a8, entries=150, sequenceid=379, filesize=12.0 K 2024-11-17T22:40:10,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/f69f6ec07d80410ba883098c99c51989 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f69f6ec07d80410ba883098c99c51989 2024-11-17T22:40:10,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883270347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883270347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,354 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f69f6ec07d80410ba883098c99c51989, entries=150, sequenceid=379, filesize=12.0 K 2024-11-17T22:40:10,355 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for dde7707c5ff08c6cad672aef50110927 in 939ms, sequenceid=379, compaction requested=true 2024-11-17T22:40:10,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:10,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:10,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-17T22:40:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-17T22:40:10,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-17T22:40:10,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0950 sec 2024-11-17T22:40:10,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.1040 sec 2024-11-17T22:40:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T22:40:10,397 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:10,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-17T22:40:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T22:40:10,401 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:10,404 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:10,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T22:40:10,552 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:10,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:40:10,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/4333fa27427c45f18549eae44ef97327 is 50, key is test_row_0/A:col10/1731883210550/Put/seqid=0 2024-11-17T22:40:10,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T22:40:10,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:10,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:10,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:10,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:10,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:10,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741918_1094 (size=14741) 2024-11-17T22:40:10,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/4333fa27427c45f18549eae44ef97327 2024-11-17T22:40:10,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/24c4077cbc24464e88bd0266710ed5b7 is 50, key is test_row_0/B:col10/1731883210550/Put/seqid=0 2024-11-17T22:40:10,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883270618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883270621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883270625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883270626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741919_1095 (size=12301) 2024-11-17T22:40:10,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T22:40:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:10,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T22:40:10,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883270737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883270737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883270737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883270747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T22:40:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:10,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:10,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883270947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883270947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883270957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883270957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T22:40:11,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T22:40:11,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:11,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/24c4077cbc24464e88bd0266710ed5b7 2024-11-17T22:40:11,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/6ec51ddb62da45f7af87a76679e270fa is 50, key is test_row_0/C:col10/1731883210550/Put/seqid=0 2024-11-17T22:40:11,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741920_1096 (size=12301) 2024-11-17T22:40:11,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T22:40:11,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:11,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883271257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883271257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883271267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883271267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T22:40:11,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:11,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/6ec51ddb62da45f7af87a76679e270fa 2024-11-17T22:40:11,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T22:40:11,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:11,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/4333fa27427c45f18549eae44ef97327 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/4333fa27427c45f18549eae44ef97327 2024-11-17T22:40:11,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:11,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/4333fa27427c45f18549eae44ef97327, entries=200, sequenceid=393, filesize=14.4 K 2024-11-17T22:40:11,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/24c4077cbc24464e88bd0266710ed5b7 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/24c4077cbc24464e88bd0266710ed5b7 2024-11-17T22:40:11,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/24c4077cbc24464e88bd0266710ed5b7, entries=150, sequenceid=393, filesize=12.0 K 2024-11-17T22:40:11,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/6ec51ddb62da45f7af87a76679e270fa as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6ec51ddb62da45f7af87a76679e270fa 2024-11-17T22:40:11,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6ec51ddb62da45f7af87a76679e270fa, entries=150, sequenceid=393, filesize=12.0 K 2024-11-17T22:40:11,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for dde7707c5ff08c6cad672aef50110927 in 959ms, sequenceid=393, compaction requested=true 2024-11-17T22:40:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:11,511 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T22:40:11,511 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:11,513 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:11,513 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/A is initiating minor compaction (all files) 2024-11-17T22:40:11,513 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/A in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,513 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/9ed7805f0eff419c95465541b185272e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/66453494ee554c3c93ec53486f9a6506, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d10878a24ff14449a97ece1a09ab5373, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/4333fa27427c45f18549eae44ef97327] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=51.2 K 2024-11-17T22:40:11,513 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:11,513 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/B is initiating minor compaction (all files) 2024-11-17T22:40:11,513 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/B in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,514 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/97f2d4d3d4b44cb1aad3a136d0e7eb9b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ac5d53404ade4697a2f6e1ab5d70ee7d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/56c685e87f5849568d9b26a3d65942a8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/24c4077cbc24464e88bd0266710ed5b7] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=48.8 K 2024-11-17T22:40:11,515 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ed7805f0eff419c95465541b185272e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731883206497 2024-11-17T22:40:11,515 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f2d4d3d4b44cb1aad3a136d0e7eb9b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731883206497 2024-11-17T22:40:11,515 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66453494ee554c3c93ec53486f9a6506, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731883206852 2024-11-17T22:40:11,515 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting ac5d53404ade4697a2f6e1ab5d70ee7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731883206852 2024-11-17T22:40:11,516 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 56c685e87f5849568d9b26a3d65942a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731883208044 2024-11-17T22:40:11,516 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting d10878a24ff14449a97ece1a09ab5373, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731883208044 2024-11-17T22:40:11,516 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 24c4077cbc24464e88bd0266710ed5b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731883210218 2024-11-17T22:40:11,516 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4333fa27427c45f18549eae44ef97327, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731883210218 2024-11-17T22:40:11,532 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:11,532 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/3a9d710f740940ff917d70eff7e57da5 is 50, key is test_row_0/A:col10/1731883210550/Put/seqid=0 2024-11-17T22:40:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T22:40:11,540 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:11,540 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/8fb74860cbce4243bdec1ee796ae6078 is 50, key is test_row_0/B:col10/1731883210550/Put/seqid=0 2024-11-17T22:40:11,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741921_1097 (size=13255) 2024-11-17T22:40:11,555 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/3a9d710f740940ff917d70eff7e57da5 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3a9d710f740940ff917d70eff7e57da5 2024-11-17T22:40:11,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741922_1098 (size=13255) 2024-11-17T22:40:11,562 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dde7707c5ff08c6cad672aef50110927/A of dde7707c5ff08c6cad672aef50110927 into 3a9d710f740940ff917d70eff7e57da5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:11,562 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:11,562 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/A, priority=12, startTime=1731883211511; duration=0sec 2024-11-17T22:40:11,563 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:11,563 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:A 2024-11-17T22:40:11,563 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:11,564 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:11,564 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/C is initiating minor compaction (all files) 2024-11-17T22:40:11,564 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/C in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,564 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/ddcccd771edb4e5eadc1565a2596b95b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/75e47df4d42c44ac95dceadcf33eb178, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f69f6ec07d80410ba883098c99c51989, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6ec51ddb62da45f7af87a76679e270fa] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=48.8 K 2024-11-17T22:40:11,565 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddcccd771edb4e5eadc1565a2596b95b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731883206497 2024-11-17T22:40:11,565 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75e47df4d42c44ac95dceadcf33eb178, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731883206852 2024-11-17T22:40:11,566 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting f69f6ec07d80410ba883098c99c51989, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731883208044 2024-11-17T22:40:11,567 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ec51ddb62da45f7af87a76679e270fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731883210218 2024-11-17T22:40:11,578 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:11,579 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/43c9e7952da2449e85fbcd8d16505120 is 50, key is test_row_0/C:col10/1731883210550/Put/seqid=0 2024-11-17T22:40:11,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741923_1099 (size=13255) 2024-11-17T22:40:11,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T22:40:11,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:11,641 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T22:40:11,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/3000777e940341d082221f34562bc859 is 50, key is test_row_0/A:col10/1731883210622/Put/seqid=0 2024-11-17T22:40:11,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741924_1100 (size=12301) 2024-11-17T22:40:11,679 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/3000777e940341d082221f34562bc859 2024-11-17T22:40:11,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/66025b1795fe499a8f32f41a6de3f80a is 50, key is test_row_0/B:col10/1731883210622/Put/seqid=0 2024-11-17T22:40:11,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741925_1101 (size=12301) 2024-11-17T22:40:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:11,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:11,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883271793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883271798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883271802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883271802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883271906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883271906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883271917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883271917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:11,965 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/8fb74860cbce4243bdec1ee796ae6078 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8fb74860cbce4243bdec1ee796ae6078 2024-11-17T22:40:11,971 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dde7707c5ff08c6cad672aef50110927/B of dde7707c5ff08c6cad672aef50110927 into 8fb74860cbce4243bdec1ee796ae6078(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:11,971 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:11,972 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/B, priority=12, startTime=1731883211511; duration=0sec 2024-11-17T22:40:11,972 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:11,972 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:B 2024-11-17T22:40:12,000 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/43c9e7952da2449e85fbcd8d16505120 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/43c9e7952da2449e85fbcd8d16505120 2024-11-17T22:40:12,006 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dde7707c5ff08c6cad672aef50110927/C of dde7707c5ff08c6cad672aef50110927 into 43c9e7952da2449e85fbcd8d16505120(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:12,006 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:12,006 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/C, priority=12, startTime=1731883211511; duration=0sec 2024-11-17T22:40:12,006 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:12,006 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:C 2024-11-17T22:40:12,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883272128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883272128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883272128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883272128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,152 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/66025b1795fe499a8f32f41a6de3f80a 2024-11-17T22:40:12,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/4e4b1085be2042efb616fdc47cb19bcc is 50, key is test_row_0/C:col10/1731883210622/Put/seqid=0 2024-11-17T22:40:12,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741926_1102 (size=12301) 2024-11-17T22:40:12,166 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/4e4b1085be2042efb616fdc47cb19bcc 2024-11-17T22:40:12,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/3000777e940341d082221f34562bc859 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3000777e940341d082221f34562bc859 2024-11-17T22:40:12,185 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3000777e940341d082221f34562bc859, entries=150, sequenceid=416, filesize=12.0 K 2024-11-17T22:40:12,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/66025b1795fe499a8f32f41a6de3f80a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/66025b1795fe499a8f32f41a6de3f80a 2024-11-17T22:40:12,194 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/66025b1795fe499a8f32f41a6de3f80a, entries=150, sequenceid=416, filesize=12.0 K 2024-11-17T22:40:12,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/4e4b1085be2042efb616fdc47cb19bcc as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/4e4b1085be2042efb616fdc47cb19bcc 2024-11-17T22:40:12,203 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/4e4b1085be2042efb616fdc47cb19bcc, entries=150, sequenceid=416, filesize=12.0 K 2024-11-17T22:40:12,205 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for dde7707c5ff08c6cad672aef50110927 in 564ms, sequenceid=416, compaction requested=false 2024-11-17T22:40:12,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:12,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:12,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-17T22:40:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-17T22:40:12,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-17T22:40:12,212 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8050 sec 2024-11-17T22:40:12,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.8150 sec 2024-11-17T22:40:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:12,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T22:40:12,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/af9a7dbfc1f44fc4a13943afa8fb3cf3 is 50, key is test_row_0/A:col10/1731883212440/Put/seqid=0 2024-11-17T22:40:12,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741927_1103 (size=12301) 2024-11-17T22:40:12,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883272462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883272464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883272465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883272467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T22:40:12,548 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:12,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-17T22:40:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T22:40:12,550 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:12,552 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:12,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:12,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883272576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883272577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883272577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883272577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T22:40:12,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-17T22:40:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:12,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:12,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883272786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883272787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883272787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883272787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:12,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-17T22:40:12,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:12,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:12,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:12,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:12,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:12,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/af9a7dbfc1f44fc4a13943afa8fb3cf3 2024-11-17T22:40:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T22:40:12,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/097763ab5f2f49178ac7d3c3de50d8f8 is 50, key is test_row_0/B:col10/1731883212440/Put/seqid=0 2024-11-17T22:40:12,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741928_1104 (size=12301) 2024-11-17T22:40:13,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-17T22:40:13,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:13,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:13,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883273096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:13,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883273097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:13,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883273097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:13,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883273097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:13,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-17T22:40:13,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:13,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T22:40:13,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/097763ab5f2f49178ac7d3c3de50d8f8 2024-11-17T22:40:13,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/10b192dc87174c04a44362a31a95d761 is 50, key is test_row_0/C:col10/1731883212440/Put/seqid=0 2024-11-17T22:40:13,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741929_1105 (size=12301) 2024-11-17T22:40:13,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-17T22:40:13,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:13,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-17T22:40:13,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:13,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:13,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883273607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883273607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:13,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883273607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883273607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:13,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-17T22:40:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:13,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/10b192dc87174c04a44362a31a95d761 2024-11-17T22:40:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T22:40:13,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/af9a7dbfc1f44fc4a13943afa8fb3cf3 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af9a7dbfc1f44fc4a13943afa8fb3cf3 2024-11-17T22:40:13,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af9a7dbfc1f44fc4a13943afa8fb3cf3, entries=150, sequenceid=438, filesize=12.0 K 2024-11-17T22:40:13,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/097763ab5f2f49178ac7d3c3de50d8f8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/097763ab5f2f49178ac7d3c3de50d8f8 2024-11-17T22:40:13,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/097763ab5f2f49178ac7d3c3de50d8f8, entries=150, sequenceid=438, filesize=12.0 K 2024-11-17T22:40:13,700 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-17T22:40:13,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/10b192dc87174c04a44362a31a95d761 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/10b192dc87174c04a44362a31a95d761 2024-11-17T22:40:13,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/10b192dc87174c04a44362a31a95d761, entries=150, sequenceid=438, filesize=12.0 K 2024-11-17T22:40:13,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for dde7707c5ff08c6cad672aef50110927 in 1267ms, sequenceid=438, compaction requested=true 2024-11-17T22:40:13,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:13,709 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:13,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:13,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:13,710 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:13,710 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:13,711 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/A is initiating minor compaction (all files) 2024-11-17T22:40:13,711 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/A in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,711 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:13,711 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/B is initiating minor compaction (all files) 2024-11-17T22:40:13,711 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3a9d710f740940ff917d70eff7e57da5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3000777e940341d082221f34562bc859, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af9a7dbfc1f44fc4a13943afa8fb3cf3] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=37.0 K 2024-11-17T22:40:13,711 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/B in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,711 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8fb74860cbce4243bdec1ee796ae6078, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/66025b1795fe499a8f32f41a6de3f80a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/097763ab5f2f49178ac7d3c3de50d8f8] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=37.0 K 2024-11-17T22:40:13,711 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a9d710f740940ff917d70eff7e57da5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731883210218 2024-11-17T22:40:13,711 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb74860cbce4243bdec1ee796ae6078, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731883210218 2024-11-17T22:40:13,711 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3000777e940341d082221f34562bc859, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731883210618 2024-11-17T22:40:13,712 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 66025b1795fe499a8f32f41a6de3f80a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731883210618 2024-11-17T22:40:13,712 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting af9a7dbfc1f44fc4a13943afa8fb3cf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731883211785 2024-11-17T22:40:13,712 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 097763ab5f2f49178ac7d3c3de50d8f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731883211785 2024-11-17T22:40:13,721 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#A#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:13,721 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/21093581c5ae44ae8410bd964fd3ae26 is 50, key is test_row_0/A:col10/1731883212440/Put/seqid=0 2024-11-17T22:40:13,721 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:13,722 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/18ed45c5e6654194953b0a8a7a4c2c8d is 50, key is test_row_0/B:col10/1731883212440/Put/seqid=0 2024-11-17T22:40:13,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741930_1106 (size=13357) 2024-11-17T22:40:13,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741931_1107 (size=13357) 2024-11-17T22:40:13,745 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/21093581c5ae44ae8410bd964fd3ae26 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/21093581c5ae44ae8410bd964fd3ae26 2024-11-17T22:40:13,748 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/18ed45c5e6654194953b0a8a7a4c2c8d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/18ed45c5e6654194953b0a8a7a4c2c8d 2024-11-17T22:40:13,755 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/A of dde7707c5ff08c6cad672aef50110927 into 21093581c5ae44ae8410bd964fd3ae26(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:13,755 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:13,755 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/A, priority=13, startTime=1731883213709; duration=0sec 2024-11-17T22:40:13,755 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:13,755 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:A 2024-11-17T22:40:13,755 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:13,758 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/B of dde7707c5ff08c6cad672aef50110927 into 18ed45c5e6654194953b0a8a7a4c2c8d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:13,758 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:13,758 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/B, priority=13, startTime=1731883213709; duration=0sec 2024-11-17T22:40:13,758 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:13,758 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:B 2024-11-17T22:40:13,759 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:13,759 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/C is initiating minor compaction (all files) 2024-11-17T22:40:13,759 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/C in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,759 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/43c9e7952da2449e85fbcd8d16505120, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/4e4b1085be2042efb616fdc47cb19bcc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/10b192dc87174c04a44362a31a95d761] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=37.0 K 2024-11-17T22:40:13,759 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43c9e7952da2449e85fbcd8d16505120, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731883210218 2024-11-17T22:40:13,760 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e4b1085be2042efb616fdc47cb19bcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731883210618 2024-11-17T22:40:13,760 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10b192dc87174c04a44362a31a95d761, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731883211785 2024-11-17T22:40:13,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-17T22:40:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:13,775 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T22:40:13,780 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/1a1a1bc742184f8b9248664aba608960 is 50, key is test_row_0/A:col10/1731883212466/Put/seqid=0 2024-11-17T22:40:13,783 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/b02100db18ab4af19a6f8d7232ff9046 is 50, key is test_row_0/C:col10/1731883212440/Put/seqid=0 2024-11-17T22:40:13,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741932_1108 (size=12301) 2024-11-17T22:40:13,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741933_1109 (size=13357) 2024-11-17T22:40:14,210 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/1a1a1bc742184f8b9248664aba608960 2024-11-17T22:40:14,217 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/b02100db18ab4af19a6f8d7232ff9046 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b02100db18ab4af19a6f8d7232ff9046 2024-11-17T22:40:14,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/bbab3972516b4c5fb6493c9c8f8f1ecb is 50, key is test_row_0/B:col10/1731883212466/Put/seqid=0 2024-11-17T22:40:14,231 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/C of dde7707c5ff08c6cad672aef50110927 into b02100db18ab4af19a6f8d7232ff9046(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:14,231 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:14,231 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/C, priority=13, startTime=1731883213710; duration=0sec 2024-11-17T22:40:14,231 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:14,231 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:C 2024-11-17T22:40:14,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741934_1110 (size=12301) 2024-11-17T22:40:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:14,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. as already flushing 2024-11-17T22:40:14,644 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/bbab3972516b4c5fb6493c9c8f8f1ecb 2024-11-17T22:40:14,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/2b631238360041e18399819e1c4d3397 is 50, key is test_row_0/C:col10/1731883212466/Put/seqid=0 2024-11-17T22:40:14,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741935_1111 (size=12301) 2024-11-17T22:40:14,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:14,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883274682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883274683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:14,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883274684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883274684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T22:40:14,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883274787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:14,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883274787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:14,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:14,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883274797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883274797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:14,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45052 deadline: 1731883274997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:15,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45000 deadline: 1731883274997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:15,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45016 deadline: 1731883275007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44978 deadline: 1731883275007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:15,057 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/2b631238360041e18399819e1c4d3397 2024-11-17T22:40:15,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/1a1a1bc742184f8b9248664aba608960 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/1a1a1bc742184f8b9248664aba608960 2024-11-17T22:40:15,068 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/1a1a1bc742184f8b9248664aba608960, entries=150, sequenceid=456, filesize=12.0 K 2024-11-17T22:40:15,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/bbab3972516b4c5fb6493c9c8f8f1ecb as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/bbab3972516b4c5fb6493c9c8f8f1ecb 2024-11-17T22:40:15,074 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/bbab3972516b4c5fb6493c9c8f8f1ecb, entries=150, sequenceid=456, filesize=12.0 K 2024-11-17T22:40:15,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/2b631238360041e18399819e1c4d3397 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/2b631238360041e18399819e1c4d3397 2024-11-17T22:40:15,084 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/2b631238360041e18399819e1c4d3397, entries=150, sequenceid=456, filesize=12.0 K 2024-11-17T22:40:15,085 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for dde7707c5ff08c6cad672aef50110927 in 1310ms, sequenceid=456, compaction requested=false 2024-11-17T22:40:15,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:15,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:15,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-17T22:40:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-17T22:40:15,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-17T22:40:15,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5350 sec 2024-11-17T22:40:15,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 2.5410 sec 2024-11-17T22:40:15,197 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-17T22:40:15,197 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:15,198 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-17T22:40:15,199 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:15,203 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,204 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:15,204 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,204 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:15,211 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-17T22:40:15,212 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:15,212 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,212 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:15,219 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-17T22:40:15,219 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:15,219 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,220 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:15,309 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:15,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T22:40:15,309 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-17T22:40:15,309 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-17T22:40:15,309 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:15,309 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:15,309 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,309 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,309 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,309 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:15,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:15,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/e9ec368c4a6b48959db99e62f6cf6488 is 50, key is test_row_0/A:col10/1731883215307/Put/seqid=0 2024-11-17T22:40:15,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741936_1112 (size=12301) 2024-11-17T22:40:15,319 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-17T22:40:15,319 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-17T22:40:15,319 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:15,319 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:15,319 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,319 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,319 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,319 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:15,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:15,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:15,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/e9ec368c4a6b48959db99e62f6cf6488 2024-11-17T22:40:15,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/f3613259f4294eb8b97f224740b2057f is 50, key is test_row_0/B:col10/1731883215307/Put/seqid=0 2024-11-17T22:40:15,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741937_1113 (size=12301) 2024-11-17T22:40:16,099 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-17T22:40:16,099 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:16,099 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:16,099 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:16,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:16,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/f3613259f4294eb8b97f224740b2057f 2024-11-17T22:40:16,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/026358180a714a9d89a122b99805bf7b is 50, key is test_row_0/C:col10/1731883215307/Put/seqid=0 2024-11-17T22:40:16,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741938_1114 (size=12301) 2024-11-17T22:40:16,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/026358180a714a9d89a122b99805bf7b 2024-11-17T22:40:16,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/e9ec368c4a6b48959db99e62f6cf6488 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/e9ec368c4a6b48959db99e62f6cf6488 2024-11-17T22:40:16,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/e9ec368c4a6b48959db99e62f6cf6488, entries=150, sequenceid=480, filesize=12.0 K 2024-11-17T22:40:16,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/f3613259f4294eb8b97f224740b2057f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/f3613259f4294eb8b97f224740b2057f 2024-11-17T22:40:16,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/f3613259f4294eb8b97f224740b2057f, entries=150, sequenceid=480, filesize=12.0 K 2024-11-17T22:40:16,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/026358180a714a9d89a122b99805bf7b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/026358180a714a9d89a122b99805bf7b 2024-11-17T22:40:16,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/026358180a714a9d89a122b99805bf7b, entries=150, sequenceid=480, filesize=12.0 K 2024-11-17T22:40:16,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=20.13 KB/20610 for dde7707c5ff08c6cad672aef50110927 in 1266ms, sequenceid=480, compaction requested=true 2024-11-17T22:40:16,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:16,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:16,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:16,576 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:16,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:16,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:16,576 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:16,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dde7707c5ff08c6cad672aef50110927:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:16,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:16,577 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:16,577 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:16,577 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/A is initiating minor compaction (all files) 2024-11-17T22:40:16,577 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dde7707c5ff08c6cad672aef50110927/B is initiating minor compaction (all files) 2024-11-17T22:40:16,577 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/A in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:16,577 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dde7707c5ff08c6cad672aef50110927/B in TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:16,577 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/18ed45c5e6654194953b0a8a7a4c2c8d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/bbab3972516b4c5fb6493c9c8f8f1ecb, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/f3613259f4294eb8b97f224740b2057f] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=37.1 K 2024-11-17T22:40:16,577 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/21093581c5ae44ae8410bd964fd3ae26, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/1a1a1bc742184f8b9248664aba608960, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/e9ec368c4a6b48959db99e62f6cf6488] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp, totalSize=37.1 K 2024-11-17T22:40:16,578 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ed45c5e6654194953b0a8a7a4c2c8d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731883211785 2024-11-17T22:40:16,578 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21093581c5ae44ae8410bd964fd3ae26, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731883211785 2024-11-17T22:40:16,578 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting bbab3972516b4c5fb6493c9c8f8f1ecb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731883212454 2024-11-17T22:40:16,578 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1a1bc742184f8b9248664aba608960, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731883212454 2024-11-17T22:40:16,578 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting f3613259f4294eb8b97f224740b2057f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731883214682 2024-11-17T22:40:16,578 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9ec368c4a6b48959db99e62f6cf6488, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731883214682 2024-11-17T22:40:16,586 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#B#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:16,586 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde7707c5ff08c6cad672aef50110927#A#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:16,586 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/56deb034f637473fa19f5204563aacda is 50, key is test_row_0/B:col10/1731883215307/Put/seqid=0 2024-11-17T22:40:16,586 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/b59036f11d224989a0357517c5d92d2c is 50, key is test_row_0/A:col10/1731883215307/Put/seqid=0 2024-11-17T22:40:16,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741939_1115 (size=13459) 2024-11-17T22:40:16,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741940_1116 (size=13459) 2024-11-17T22:40:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T22:40:16,718 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3400 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3321 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1526 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4576 rows 2024-11-17T22:40:16,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1529 2024-11-17T22:40:16,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4587 rows 2024-11-17T22:40:16,719 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T22:40:16,719 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T22:40:16,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:16,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:16,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:16,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T22:40:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:16,730 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883216730"}]},"ts":"1731883216730"} 2024-11-17T22:40:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T22:40:16,732 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T22:40:16,732 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T22:40:16,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T22:40:16,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde7707c5ff08c6cad672aef50110927, UNASSIGN}] 2024-11-17T22:40:16,739 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde7707c5ff08c6cad672aef50110927, UNASSIGN 2024-11-17T22:40:16,740 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=dde7707c5ff08c6cad672aef50110927, regionState=CLOSING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:16,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde7707c5ff08c6cad672aef50110927, UNASSIGN because future has completed 2024-11-17T22:40:16,743 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T22:40:16,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:40:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T22:40:16,903 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:16,903 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T22:40:16,905 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing dde7707c5ff08c6cad672aef50110927, disabling compactions & flushes 2024-11-17T22:40:16,905 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:17,003 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/56deb034f637473fa19f5204563aacda as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/56deb034f637473fa19f5204563aacda 2024-11-17T22:40:17,003 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/b59036f11d224989a0357517c5d92d2c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b59036f11d224989a0357517c5d92d2c 2024-11-17T22:40:17,009 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/A of dde7707c5ff08c6cad672aef50110927 into b59036f11d224989a0357517c5d92d2c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:17,009 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dde7707c5ff08c6cad672aef50110927/B of dde7707c5ff08c6cad672aef50110927 into 56deb034f637473fa19f5204563aacda(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:17,009 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:17,009 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dde7707c5ff08c6cad672aef50110927: 2024-11-17T22:40:17,009 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/A, priority=13, startTime=1731883216576; duration=0sec 2024-11-17T22:40:17,009 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:17,009 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., storeName=dde7707c5ff08c6cad672aef50110927/B, priority=13, startTime=1731883216576; duration=0sec 2024-11-17T22:40:17,009 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:17,009 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:17,009 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:17,009 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:A 2024-11-17T22:40:17,009 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:B 2024-11-17T22:40:17,009 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. after waiting 0 ms 2024-11-17T22:40:17,009 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:17,009 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing dde7707c5ff08c6cad672aef50110927 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-17T22:40:17,009 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. because compaction request was cancelled 2024-11-17T22:40:17,009 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde7707c5ff08c6cad672aef50110927:C 2024-11-17T22:40:17,013 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/7a9da0abcfa8489cb7cad573ed01e1b7 is 50, key is test_row_0/A:col10/1731883215318/Put/seqid=0 2024-11-17T22:40:17,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741941_1117 (size=9857) 2024-11-17T22:40:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T22:40:17,245 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T22:40:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T22:40:17,419 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/7a9da0abcfa8489cb7cad573ed01e1b7 2024-11-17T22:40:17,429 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/6c089f5008294c2db9ac14d4ac76af89 is 50, key is test_row_0/B:col10/1731883215318/Put/seqid=0 2024-11-17T22:40:17,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741942_1118 (size=9857) 2024-11-17T22:40:17,835 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/6c089f5008294c2db9ac14d4ac76af89 2024-11-17T22:40:17,847 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/fa3c76af6d2f4f6293e34353c26eea83 is 50, key is test_row_0/C:col10/1731883215318/Put/seqid=0 2024-11-17T22:40:17,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741943_1119 (size=9857) 2024-11-17T22:40:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T22:40:18,252 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/fa3c76af6d2f4f6293e34353c26eea83 2024-11-17T22:40:18,258 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/A/7a9da0abcfa8489cb7cad573ed01e1b7 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7a9da0abcfa8489cb7cad573ed01e1b7 2024-11-17T22:40:18,265 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7a9da0abcfa8489cb7cad573ed01e1b7, entries=100, sequenceid=488, filesize=9.6 K 2024-11-17T22:40:18,266 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/B/6c089f5008294c2db9ac14d4ac76af89 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/6c089f5008294c2db9ac14d4ac76af89 2024-11-17T22:40:18,271 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/6c089f5008294c2db9ac14d4ac76af89, entries=100, sequenceid=488, filesize=9.6 K 2024-11-17T22:40:18,272 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/.tmp/C/fa3c76af6d2f4f6293e34353c26eea83 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/fa3c76af6d2f4f6293e34353c26eea83 2024-11-17T22:40:18,278 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/fa3c76af6d2f4f6293e34353c26eea83, entries=100, sequenceid=488, filesize=9.6 K 2024-11-17T22:40:18,279 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for dde7707c5ff08c6cad672aef50110927 in 1270ms, sequenceid=488, compaction requested=true 2024-11-17T22:40:18,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2fffadd4ec124ffc832f3062aa114f63, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/741c2e376f5348528da06b9e89aeb878, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2f44c4a3f9ac47168e2ee79d8d10d995, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/ae3cc259bcf84049aabd2451a574a984, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2d60058b31c64ec195d5ca3dce6f3a04, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/ae42126497f1494f8793a6c8214affe8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/9c2747e3527d4c11b7c89c57ac7d2243, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af2c3b3c503c4b7f9117826baa5c7044, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7ef5c4306f73489d96dfbe31d85708c6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d306b0bd5ea546859ec9a355f14976b1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/c14a7f58d8334d89b2a83635a3e774d6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/78d6c9b6dae9423c982f10d390da8f03, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/bde7855d25e14fc580858c7815a7357f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3f499750a4894d4da3e7942bc2f5a299, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/737311e649b74e4996b3f5ac951c40e1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/08a8fd2ea65d4de19103c2300215c1f2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b3d6bf5f985e438bae2f6ce0fc5632f7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b63166813ab244bab93afb0c5ed76a2a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/e1b9105fd0854c1e979b62bed40db04e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/5d7cab0c049d4046a1d0654a0dc043a2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7857546d7a7a4c738978a92ada163c25, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/4bab787366664ec3b18c9533a3edd226, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/a2e296caffb646e1bba5c0e926a0eb71, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/89c098bb551040a0afa538623836791a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/9ed7805f0eff419c95465541b185272e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/66453494ee554c3c93ec53486f9a6506, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d10878a24ff14449a97ece1a09ab5373, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/4333fa27427c45f18549eae44ef97327, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3a9d710f740940ff917d70eff7e57da5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3000777e940341d082221f34562bc859, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/21093581c5ae44ae8410bd964fd3ae26, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af9a7dbfc1f44fc4a13943afa8fb3cf3, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/1a1a1bc742184f8b9248664aba608960, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/e9ec368c4a6b48959db99e62f6cf6488] to archive 2024-11-17T22:40:18,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:40:18,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2fffadd4ec124ffc832f3062aa114f63 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2fffadd4ec124ffc832f3062aa114f63 2024-11-17T22:40:18,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/741c2e376f5348528da06b9e89aeb878 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/741c2e376f5348528da06b9e89aeb878 2024-11-17T22:40:18,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2f44c4a3f9ac47168e2ee79d8d10d995 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2f44c4a3f9ac47168e2ee79d8d10d995 2024-11-17T22:40:18,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/ae3cc259bcf84049aabd2451a574a984 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/ae3cc259bcf84049aabd2451a574a984 2024-11-17T22:40:18,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2d60058b31c64ec195d5ca3dce6f3a04 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/2d60058b31c64ec195d5ca3dce6f3a04 2024-11-17T22:40:18,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/ae42126497f1494f8793a6c8214affe8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/ae42126497f1494f8793a6c8214affe8 2024-11-17T22:40:18,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/9c2747e3527d4c11b7c89c57ac7d2243 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/9c2747e3527d4c11b7c89c57ac7d2243 2024-11-17T22:40:18,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af2c3b3c503c4b7f9117826baa5c7044 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af2c3b3c503c4b7f9117826baa5c7044 2024-11-17T22:40:18,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7ef5c4306f73489d96dfbe31d85708c6 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7ef5c4306f73489d96dfbe31d85708c6 2024-11-17T22:40:18,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d306b0bd5ea546859ec9a355f14976b1 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d306b0bd5ea546859ec9a355f14976b1 2024-11-17T22:40:18,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/c14a7f58d8334d89b2a83635a3e774d6 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/c14a7f58d8334d89b2a83635a3e774d6 2024-11-17T22:40:18,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/78d6c9b6dae9423c982f10d390da8f03 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/78d6c9b6dae9423c982f10d390da8f03 2024-11-17T22:40:18,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/bde7855d25e14fc580858c7815a7357f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/bde7855d25e14fc580858c7815a7357f 2024-11-17T22:40:18,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3f499750a4894d4da3e7942bc2f5a299 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3f499750a4894d4da3e7942bc2f5a299 2024-11-17T22:40:18,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/737311e649b74e4996b3f5ac951c40e1 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/737311e649b74e4996b3f5ac951c40e1 2024-11-17T22:40:18,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/08a8fd2ea65d4de19103c2300215c1f2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/08a8fd2ea65d4de19103c2300215c1f2 2024-11-17T22:40:18,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b3d6bf5f985e438bae2f6ce0fc5632f7 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b3d6bf5f985e438bae2f6ce0fc5632f7 2024-11-17T22:40:18,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b63166813ab244bab93afb0c5ed76a2a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b63166813ab244bab93afb0c5ed76a2a 2024-11-17T22:40:18,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/e1b9105fd0854c1e979b62bed40db04e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/e1b9105fd0854c1e979b62bed40db04e 2024-11-17T22:40:18,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/5d7cab0c049d4046a1d0654a0dc043a2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/5d7cab0c049d4046a1d0654a0dc043a2 2024-11-17T22:40:18,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7857546d7a7a4c738978a92ada163c25 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7857546d7a7a4c738978a92ada163c25 2024-11-17T22:40:18,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/4bab787366664ec3b18c9533a3edd226 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/4bab787366664ec3b18c9533a3edd226 2024-11-17T22:40:18,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/a2e296caffb646e1bba5c0e926a0eb71 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/a2e296caffb646e1bba5c0e926a0eb71 2024-11-17T22:40:18,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/89c098bb551040a0afa538623836791a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/89c098bb551040a0afa538623836791a 2024-11-17T22:40:18,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/9ed7805f0eff419c95465541b185272e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/9ed7805f0eff419c95465541b185272e 2024-11-17T22:40:18,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/66453494ee554c3c93ec53486f9a6506 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/66453494ee554c3c93ec53486f9a6506 2024-11-17T22:40:18,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d10878a24ff14449a97ece1a09ab5373 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/d10878a24ff14449a97ece1a09ab5373 2024-11-17T22:40:18,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/4333fa27427c45f18549eae44ef97327 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/4333fa27427c45f18549eae44ef97327 2024-11-17T22:40:18,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3a9d710f740940ff917d70eff7e57da5 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3a9d710f740940ff917d70eff7e57da5 2024-11-17T22:40:18,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3000777e940341d082221f34562bc859 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/3000777e940341d082221f34562bc859 2024-11-17T22:40:18,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/21093581c5ae44ae8410bd964fd3ae26 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/21093581c5ae44ae8410bd964fd3ae26 2024-11-17T22:40:18,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af9a7dbfc1f44fc4a13943afa8fb3cf3 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/af9a7dbfc1f44fc4a13943afa8fb3cf3 2024-11-17T22:40:18,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/1a1a1bc742184f8b9248664aba608960 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/1a1a1bc742184f8b9248664aba608960 2024-11-17T22:40:18,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/e9ec368c4a6b48959db99e62f6cf6488 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/e9ec368c4a6b48959db99e62f6cf6488 2024-11-17T22:40:18,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/a9f4487f660340fcac9d5df9d363c99b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ad9e879fb4e9430690f0fb670b9b781b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/482f0dde04084951bab14c9944876b78, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/c220fd4de64d4d4387d8c47750b94383, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/5cce1f43d4914273b4d2bc31f66d9a99, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/dd75aa14d8ff4f7c9840c9eeff25bf47, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8cf936d1dcb3401a8fa55d95196391fb, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/869f5ec04eec471c8dc31bb1f28d947f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/14b039f7ece44cc3b925800ca4f97233, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e3498b00009d4609b242b41b15410d18, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/626a86fe61b544bca47543e2b5486b5a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/c2053f4c2a4d4f29a64edd40826b6216, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/484780c37f804de8a41791ee01d7cdb6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e0bee3ce4b624592bcce2f9f1570dfd4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8935387dc3bc4258a4563cff65bd145a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/936a0394602a4f21a1a212d3741fb503, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/3e3c5ebbb84349078ba5417496d2d122, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/cdf3aec9b9644d9daba77c470e2067e2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/00743dccc9a34b48b040a29ae4828151, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/4bdb261d4b484a7cb4e18997cf5cff86, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/d688fd1329524d36825b2d3b67fec487, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/0a582bc15fda42f198c8d13763410b6b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/db323a99c3d34db3ace787d9e03d78a4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/97f2d4d3d4b44cb1aad3a136d0e7eb9b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/aa191f10ce5e4274aab1e70cef635373, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ac5d53404ade4697a2f6e1ab5d70ee7d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/56c685e87f5849568d9b26a3d65942a8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8fb74860cbce4243bdec1ee796ae6078, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/24c4077cbc24464e88bd0266710ed5b7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/66025b1795fe499a8f32f41a6de3f80a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/18ed45c5e6654194953b0a8a7a4c2c8d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/097763ab5f2f49178ac7d3c3de50d8f8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/bbab3972516b4c5fb6493c9c8f8f1ecb, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/f3613259f4294eb8b97f224740b2057f] to archive 2024-11-17T22:40:18,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:40:18,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/a9f4487f660340fcac9d5df9d363c99b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/a9f4487f660340fcac9d5df9d363c99b 2024-11-17T22:40:18,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ad9e879fb4e9430690f0fb670b9b781b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ad9e879fb4e9430690f0fb670b9b781b 2024-11-17T22:40:18,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/482f0dde04084951bab14c9944876b78 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/482f0dde04084951bab14c9944876b78 2024-11-17T22:40:18,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/c220fd4de64d4d4387d8c47750b94383 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/c220fd4de64d4d4387d8c47750b94383 2024-11-17T22:40:18,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/5cce1f43d4914273b4d2bc31f66d9a99 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/5cce1f43d4914273b4d2bc31f66d9a99 2024-11-17T22:40:18,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/dd75aa14d8ff4f7c9840c9eeff25bf47 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/dd75aa14d8ff4f7c9840c9eeff25bf47 2024-11-17T22:40:18,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8cf936d1dcb3401a8fa55d95196391fb to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8cf936d1dcb3401a8fa55d95196391fb 2024-11-17T22:40:18,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/869f5ec04eec471c8dc31bb1f28d947f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/869f5ec04eec471c8dc31bb1f28d947f 2024-11-17T22:40:18,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/14b039f7ece44cc3b925800ca4f97233 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/14b039f7ece44cc3b925800ca4f97233 2024-11-17T22:40:18,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e3498b00009d4609b242b41b15410d18 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e3498b00009d4609b242b41b15410d18 2024-11-17T22:40:18,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/626a86fe61b544bca47543e2b5486b5a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/626a86fe61b544bca47543e2b5486b5a 2024-11-17T22:40:18,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/c2053f4c2a4d4f29a64edd40826b6216 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/c2053f4c2a4d4f29a64edd40826b6216 2024-11-17T22:40:18,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/484780c37f804de8a41791ee01d7cdb6 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/484780c37f804de8a41791ee01d7cdb6 2024-11-17T22:40:18,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e0bee3ce4b624592bcce2f9f1570dfd4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/e0bee3ce4b624592bcce2f9f1570dfd4 2024-11-17T22:40:18,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8935387dc3bc4258a4563cff65bd145a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8935387dc3bc4258a4563cff65bd145a 2024-11-17T22:40:18,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/936a0394602a4f21a1a212d3741fb503 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/936a0394602a4f21a1a212d3741fb503 2024-11-17T22:40:18,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/3e3c5ebbb84349078ba5417496d2d122 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/3e3c5ebbb84349078ba5417496d2d122 2024-11-17T22:40:18,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/cdf3aec9b9644d9daba77c470e2067e2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/cdf3aec9b9644d9daba77c470e2067e2 2024-11-17T22:40:18,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/00743dccc9a34b48b040a29ae4828151 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/00743dccc9a34b48b040a29ae4828151 2024-11-17T22:40:18,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/4bdb261d4b484a7cb4e18997cf5cff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/4bdb261d4b484a7cb4e18997cf5cff86 2024-11-17T22:40:18,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/d688fd1329524d36825b2d3b67fec487 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/d688fd1329524d36825b2d3b67fec487 2024-11-17T22:40:18,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/0a582bc15fda42f198c8d13763410b6b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/0a582bc15fda42f198c8d13763410b6b 2024-11-17T22:40:18,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/db323a99c3d34db3ace787d9e03d78a4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/db323a99c3d34db3ace787d9e03d78a4 2024-11-17T22:40:18,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/97f2d4d3d4b44cb1aad3a136d0e7eb9b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/97f2d4d3d4b44cb1aad3a136d0e7eb9b 2024-11-17T22:40:18,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/aa191f10ce5e4274aab1e70cef635373 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/aa191f10ce5e4274aab1e70cef635373 2024-11-17T22:40:18,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ac5d53404ade4697a2f6e1ab5d70ee7d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/ac5d53404ade4697a2f6e1ab5d70ee7d 2024-11-17T22:40:18,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/56c685e87f5849568d9b26a3d65942a8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/56c685e87f5849568d9b26a3d65942a8 2024-11-17T22:40:18,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8fb74860cbce4243bdec1ee796ae6078 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/8fb74860cbce4243bdec1ee796ae6078 2024-11-17T22:40:18,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/24c4077cbc24464e88bd0266710ed5b7 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/24c4077cbc24464e88bd0266710ed5b7 2024-11-17T22:40:18,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/66025b1795fe499a8f32f41a6de3f80a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/66025b1795fe499a8f32f41a6de3f80a 2024-11-17T22:40:18,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/18ed45c5e6654194953b0a8a7a4c2c8d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/18ed45c5e6654194953b0a8a7a4c2c8d 2024-11-17T22:40:18,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/097763ab5f2f49178ac7d3c3de50d8f8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/097763ab5f2f49178ac7d3c3de50d8f8 2024-11-17T22:40:18,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/bbab3972516b4c5fb6493c9c8f8f1ecb to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/bbab3972516b4c5fb6493c9c8f8f1ecb 2024-11-17T22:40:18,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/f3613259f4294eb8b97f224740b2057f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/f3613259f4294eb8b97f224740b2057f 2024-11-17T22:40:18,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b91ec837a302448dbfed0312fa31f395, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/5bc58d5f84d24958a8551d32a3b017d2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/214ee6f0fbcb421bb58a69727f21372e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6b57c966629c45ed8346472f00451a63, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/d6a3250dc0bc41a288f61f85063e78dd, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/34f3e6f386814222818069055369fe1a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f56542f3159940918ea1e8c5ea4a77c0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/41a03858a43341c1adbf6eb9ebef551a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/22ed90edc1644f59824880be946ddd5a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/0df011f8ba2a4173893bce9f1a953e61, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/665d6ab65b4f4ba49b329170dcd091f5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/fc750c18784e46b190f6b7e8dd8628d9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/45b3c56ebb27481d836c7c9854638cac, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3d88b187d9094dd081f2e56ca9d4689f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/0e1f7eb1b5b8430c94d2136e6f944107, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b8056e849a3545a28a3ba40d031810ce, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3a47e5c9ebee4b89bd6763b1addfaf81, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/eaba7d174ada4d23b3a41c0c5feac042, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/16a33d44d63a442c8378738216eedeca, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/a4af620f37dd4a939d7371f7338de191, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/235fd52f323448b182070533420741ef, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/1dd66a17399a402683373a3be3773a40, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/63f38a18ff4741fa82586c34cfe5743a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/ddcccd771edb4e5eadc1565a2596b95b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f2abced56d384340b5928b48c4217c67, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/75e47df4d42c44ac95dceadcf33eb178, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f69f6ec07d80410ba883098c99c51989, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/43c9e7952da2449e85fbcd8d16505120, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6ec51ddb62da45f7af87a76679e270fa, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/4e4b1085be2042efb616fdc47cb19bcc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/10b192dc87174c04a44362a31a95d761] to archive 2024-11-17T22:40:18,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:40:18,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b91ec837a302448dbfed0312fa31f395 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b91ec837a302448dbfed0312fa31f395 2024-11-17T22:40:18,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/5bc58d5f84d24958a8551d32a3b017d2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/5bc58d5f84d24958a8551d32a3b017d2 2024-11-17T22:40:18,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/214ee6f0fbcb421bb58a69727f21372e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/214ee6f0fbcb421bb58a69727f21372e 2024-11-17T22:40:18,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6b57c966629c45ed8346472f00451a63 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6b57c966629c45ed8346472f00451a63 2024-11-17T22:40:18,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/d6a3250dc0bc41a288f61f85063e78dd to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/d6a3250dc0bc41a288f61f85063e78dd 2024-11-17T22:40:18,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/34f3e6f386814222818069055369fe1a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/34f3e6f386814222818069055369fe1a 2024-11-17T22:40:18,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f56542f3159940918ea1e8c5ea4a77c0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f56542f3159940918ea1e8c5ea4a77c0 2024-11-17T22:40:18,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/41a03858a43341c1adbf6eb9ebef551a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/41a03858a43341c1adbf6eb9ebef551a 2024-11-17T22:40:18,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/22ed90edc1644f59824880be946ddd5a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/22ed90edc1644f59824880be946ddd5a 2024-11-17T22:40:18,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/0df011f8ba2a4173893bce9f1a953e61 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/0df011f8ba2a4173893bce9f1a953e61 2024-11-17T22:40:18,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/665d6ab65b4f4ba49b329170dcd091f5 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/665d6ab65b4f4ba49b329170dcd091f5 2024-11-17T22:40:18,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/fc750c18784e46b190f6b7e8dd8628d9 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/fc750c18784e46b190f6b7e8dd8628d9 2024-11-17T22:40:18,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/45b3c56ebb27481d836c7c9854638cac to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/45b3c56ebb27481d836c7c9854638cac 2024-11-17T22:40:18,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3d88b187d9094dd081f2e56ca9d4689f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3d88b187d9094dd081f2e56ca9d4689f 2024-11-17T22:40:18,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/0e1f7eb1b5b8430c94d2136e6f944107 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/0e1f7eb1b5b8430c94d2136e6f944107 2024-11-17T22:40:18,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b8056e849a3545a28a3ba40d031810ce to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b8056e849a3545a28a3ba40d031810ce 2024-11-17T22:40:18,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3a47e5c9ebee4b89bd6763b1addfaf81 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/3a47e5c9ebee4b89bd6763b1addfaf81 2024-11-17T22:40:18,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/eaba7d174ada4d23b3a41c0c5feac042 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/eaba7d174ada4d23b3a41c0c5feac042 2024-11-17T22:40:18,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/16a33d44d63a442c8378738216eedeca to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/16a33d44d63a442c8378738216eedeca 2024-11-17T22:40:18,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/a4af620f37dd4a939d7371f7338de191 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/a4af620f37dd4a939d7371f7338de191 2024-11-17T22:40:18,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/235fd52f323448b182070533420741ef to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/235fd52f323448b182070533420741ef 2024-11-17T22:40:18,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/1dd66a17399a402683373a3be3773a40 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/1dd66a17399a402683373a3be3773a40 2024-11-17T22:40:18,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/63f38a18ff4741fa82586c34cfe5743a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/63f38a18ff4741fa82586c34cfe5743a 2024-11-17T22:40:18,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/ddcccd771edb4e5eadc1565a2596b95b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/ddcccd771edb4e5eadc1565a2596b95b 2024-11-17T22:40:18,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f2abced56d384340b5928b48c4217c67 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f2abced56d384340b5928b48c4217c67 2024-11-17T22:40:18,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/75e47df4d42c44ac95dceadcf33eb178 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/75e47df4d42c44ac95dceadcf33eb178 2024-11-17T22:40:18,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f69f6ec07d80410ba883098c99c51989 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/f69f6ec07d80410ba883098c99c51989 2024-11-17T22:40:18,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/43c9e7952da2449e85fbcd8d16505120 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/43c9e7952da2449e85fbcd8d16505120 2024-11-17T22:40:18,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6ec51ddb62da45f7af87a76679e270fa to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/6ec51ddb62da45f7af87a76679e270fa 2024-11-17T22:40:18,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/4e4b1085be2042efb616fdc47cb19bcc to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/4e4b1085be2042efb616fdc47cb19bcc 2024-11-17T22:40:18,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/10b192dc87174c04a44362a31a95d761 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/10b192dc87174c04a44362a31a95d761 2024-11-17T22:40:18,429 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/recovered.edits/491.seqid, newMaxSeqId=491, maxSeqId=1 2024-11-17T22:40:18,430 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. 2024-11-17T22:40:18,430 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for dde7707c5ff08c6cad672aef50110927: Waiting for close lock at 1731883216905Running coprocessor pre-close hooks at 1731883216905Disabling compacts and flushes for region at 1731883216905Disabling writes for close at 1731883217009 (+104 ms)Obtaining lock to block concurrent updates at 1731883217009Preparing flush snapshotting stores in dde7707c5ff08c6cad672aef50110927 at 1731883217009Finished memstore snapshotting TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731883217010 (+1 ms)Flushing stores of TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927. at 1731883217010Flushing dde7707c5ff08c6cad672aef50110927/A: creating writer at 1731883217010Flushing dde7707c5ff08c6cad672aef50110927/A: appending metadata at 1731883217013 (+3 ms)Flushing dde7707c5ff08c6cad672aef50110927/A: closing flushed file at 1731883217013Flushing dde7707c5ff08c6cad672aef50110927/B: creating writer at 1731883217425 (+412 ms)Flushing dde7707c5ff08c6cad672aef50110927/B: appending metadata at 1731883217428 (+3 ms)Flushing dde7707c5ff08c6cad672aef50110927/B: closing flushed file at 1731883217429 (+1 ms)Flushing dde7707c5ff08c6cad672aef50110927/C: creating writer at 1731883217843 (+414 ms)Flushing dde7707c5ff08c6cad672aef50110927/C: appending metadata at 1731883217846 (+3 ms)Flushing dde7707c5ff08c6cad672aef50110927/C: closing flushed file at 1731883217846Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33e2f6b2: reopening flushed file at 1731883218257 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b6dcd34: reopening flushed file at 1731883218265 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50955ad8: reopening flushed file at 1731883218271 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for dde7707c5ff08c6cad672aef50110927 in 1270ms, sequenceid=488, compaction requested=true at 1731883218279 (+8 ms)Writing region close event to WAL at 1731883218425 (+146 ms)Running coprocessor post-close hooks at 1731883218429 (+4 ms)Closed at 1731883218429 2024-11-17T22:40:18,433 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:18,433 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=dde7707c5ff08c6cad672aef50110927, regionState=CLOSED 2024-11-17T22:40:18,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:40:18,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-17T22:40:18,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure dde7707c5ff08c6cad672aef50110927, server=1a6e40b21a48,43011,1731883188895 in 1.6940 sec 2024-11-17T22:40:18,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-17T22:40:18,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dde7707c5ff08c6cad672aef50110927, UNASSIGN in 1.7010 sec 2024-11-17T22:40:18,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-17T22:40:18,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7090 sec 2024-11-17T22:40:18,446 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883218446"}]},"ts":"1731883218446"} 2024-11-17T22:40:18,448 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T22:40:18,448 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T22:40:18,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7250 sec 2024-11-17T22:40:18,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T22:40:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T22:40:18,878 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:18,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T22:40:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:18,891 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-17T22:40:18,892 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:18,897 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:18,901 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/recovered.edits] 2024-11-17T22:40:18,904 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7a9da0abcfa8489cb7cad573ed01e1b7 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/7a9da0abcfa8489cb7cad573ed01e1b7 2024-11-17T22:40:18,905 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b59036f11d224989a0357517c5d92d2c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/A/b59036f11d224989a0357517c5d92d2c 2024-11-17T22:40:18,908 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/56deb034f637473fa19f5204563aacda to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/56deb034f637473fa19f5204563aacda 2024-11-17T22:40:18,909 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/6c089f5008294c2db9ac14d4ac76af89 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/B/6c089f5008294c2db9ac14d4ac76af89 2024-11-17T22:40:18,911 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/026358180a714a9d89a122b99805bf7b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/026358180a714a9d89a122b99805bf7b 2024-11-17T22:40:18,912 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/2b631238360041e18399819e1c4d3397 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/2b631238360041e18399819e1c4d3397 2024-11-17T22:40:18,914 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b02100db18ab4af19a6f8d7232ff9046 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/b02100db18ab4af19a6f8d7232ff9046 2024-11-17T22:40:18,915 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/fa3c76af6d2f4f6293e34353c26eea83 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/C/fa3c76af6d2f4f6293e34353c26eea83 2024-11-17T22:40:18,918 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/recovered.edits/491.seqid to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927/recovered.edits/491.seqid 2024-11-17T22:40:18,918 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dde7707c5ff08c6cad672aef50110927 2024-11-17T22:40:18,918 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T22:40:18,923 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:18,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-17T22:40:18,931 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T22:40:18,962 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T22:40:18,964 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:18,964 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T22:40:18,965 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731883218964"}]},"ts":"9223372036854775807"} 2024-11-17T22:40:18,970 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T22:40:18,970 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => dde7707c5ff08c6cad672aef50110927, NAME => 'TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T22:40:18,971 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T22:40:18,971 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731883218971"}]},"ts":"9223372036854775807"} 2024-11-17T22:40:18,975 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T22:40:18,976 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:18,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 92 msec 2024-11-17T22:40:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-17T22:40:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T22:40:18,998 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:19,009 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-39089393_22 at /127.0.0.1:55504 [Waiting for operation #342] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;1a6e40b21a48:43011-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-225167225_22 at /127.0.0.1:55634 [Waiting for operation #325] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=371 (was 311) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4367 (was 5299) 2024-11-17T22:40:19,018 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=371, ProcessCount=11, AvailableMemoryMB=4366 2024-11-17T22:40:19,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T22:40:19,020 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T22:40:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:19,022 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T22:40:19,022 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:19,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-17T22:40:19,024 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T22:40:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-17T22:40:19,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741944_1120 (size=959) 2024-11-17T22:40:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-17T22:40:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-17T22:40:19,435 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71 2024-11-17T22:40:19,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741945_1121 (size=53) 2024-11-17T22:40:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-17T22:40:19,845 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:40:19,846 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3893ec8e69968d25a1df1a69563cc10b, disabling compactions & flushes 2024-11-17T22:40:19,846 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:19,846 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:19,846 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. after waiting 0 ms 2024-11-17T22:40:19,846 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:19,846 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:19,846 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3893ec8e69968d25a1df1a69563cc10b: Waiting for close lock at 1731883219845Disabling compacts and flushes for region at 1731883219845Disabling writes for close at 1731883219846 (+1 ms)Writing region close event to WAL at 1731883219846Closed at 1731883219846 2024-11-17T22:40:19,849 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T22:40:19,849 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731883219849"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731883219849"}]},"ts":"1731883219849"} 2024-11-17T22:40:19,852 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T22:40:19,853 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T22:40:19,853 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883219853"}]},"ts":"1731883219853"} 2024-11-17T22:40:19,855 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T22:40:19,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, ASSIGN}] 2024-11-17T22:40:19,857 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, ASSIGN 2024-11-17T22:40:19,858 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, ASSIGN; state=OFFLINE, location=1a6e40b21a48,43011,1731883188895; forceNewPlan=false, retain=false 2024-11-17T22:40:20,009 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=3893ec8e69968d25a1df1a69563cc10b, regionState=OPENING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:20,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, ASSIGN because future has completed 2024-11-17T22:40:20,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:40:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-17T22:40:20,169 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:20,169 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} 2024-11-17T22:40:20,170 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,170 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:40:20,170 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,170 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,173 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,177 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3893ec8e69968d25a1df1a69563cc10b columnFamilyName A 2024-11-17T22:40:20,177 DEBUG [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:20,178 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(327): Store=3893ec8e69968d25a1df1a69563cc10b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:40:20,178 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,180 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3893ec8e69968d25a1df1a69563cc10b columnFamilyName B 2024-11-17T22:40:20,181 DEBUG [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:20,181 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(327): Store=3893ec8e69968d25a1df1a69563cc10b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:40:20,182 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,184 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3893ec8e69968d25a1df1a69563cc10b columnFamilyName C 2024-11-17T22:40:20,184 DEBUG [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:20,185 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(327): Store=3893ec8e69968d25a1df1a69563cc10b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:40:20,185 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,186 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,187 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,188 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,188 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,189 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T22:40:20,191 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,193 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T22:40:20,194 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 3893ec8e69968d25a1df1a69563cc10b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69613082, jitterRate=0.03731575608253479}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T22:40:20,194 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:20,196 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 3893ec8e69968d25a1df1a69563cc10b: Running coprocessor pre-open hook at 1731883220171Writing region info on filesystem at 1731883220171Initializing all the Stores at 1731883220172 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883220172Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883220172Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883220172Cleaning up temporary data from old regions at 1731883220189 (+17 ms)Running coprocessor post-open hooks at 1731883220194 (+5 ms)Region opened successfully at 1731883220196 (+2 ms) 2024-11-17T22:40:20,196 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., pid=38, masterSystemTime=1731883220163 2024-11-17T22:40:20,198 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:20,198 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:20,199 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=3893ec8e69968d25a1df1a69563cc10b, regionState=OPEN, openSeqNum=2, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:20,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:40:20,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-17T22:40:20,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 in 190 msec 2024-11-17T22:40:20,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-17T22:40:20,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, ASSIGN in 348 msec 2024-11-17T22:40:20,206 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T22:40:20,206 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883220206"}]},"ts":"1731883220206"} 2024-11-17T22:40:20,207 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T22:40:20,208 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T22:40:20,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-11-17T22:40:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-17T22:40:21,178 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:21,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:21,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:21,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:21,183 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6746317a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:21,185 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12afde2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:21,187 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:21,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:21,189 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:21,191 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T22:40:21,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a6e40b21a48,45671,1731883188234 2024-11-17T22:40:21,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@302feb9a 2024-11-17T22:40:21,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T22:40:21,194 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T22:40:21,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T22:40:21,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T22:40:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:21,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741946_1122 (size=995) 2024-11-17T22:40:21,629 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T22:40:21,629 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T22:40:21,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T22:40:21,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, REOPEN/MOVE}] 2024-11-17T22:40:21,641 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, REOPEN/MOVE 2024-11-17T22:40:21,642 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=3893ec8e69968d25a1df1a69563cc10b, regionState=CLOSING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:21,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, REOPEN/MOVE because future has completed 2024-11-17T22:40:21,644 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T22:40:21,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:40:21,796 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:21,797 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T22:40:21,797 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing 3893ec8e69968d25a1df1a69563cc10b, disabling compactions & flushes 2024-11-17T22:40:21,797 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:21,797 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:21,797 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. after waiting 0 ms 2024-11-17T22:40:21,797 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:21,801 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T22:40:21,802 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:21,802 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for 3893ec8e69968d25a1df1a69563cc10b: Waiting for close lock at 1731883221797Running coprocessor pre-close hooks at 1731883221797Disabling compacts and flushes for region at 1731883221797Disabling writes for close at 1731883221797Writing region close event to WAL at 1731883221797Running coprocessor post-close hooks at 1731883221802 (+5 ms)Closed at 1731883221802 2024-11-17T22:40:21,802 WARN [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: 3893ec8e69968d25a1df1a69563cc10b to self. 2024-11-17T22:40:21,804 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:21,805 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=3893ec8e69968d25a1df1a69563cc10b, regionState=CLOSED 2024-11-17T22:40:21,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:40:21,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-17T22:40:21,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 in 164 msec 2024-11-17T22:40:21,810 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, REOPEN/MOVE; state=CLOSED, location=1a6e40b21a48,43011,1731883188895; forceNewPlan=false, retain=true 2024-11-17T22:40:21,961 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=3893ec8e69968d25a1df1a69563cc10b, regionState=OPENING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:21,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, REOPEN/MOVE because future has completed 2024-11-17T22:40:21,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:40:22,118 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,118 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} 2024-11-17T22:40:22,119 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,119 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:40:22,119 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,119 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,123 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,129 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3893ec8e69968d25a1df1a69563cc10b columnFamilyName A 2024-11-17T22:40:22,131 DEBUG [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:22,132 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(327): Store=3893ec8e69968d25a1df1a69563cc10b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:40:22,132 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,133 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3893ec8e69968d25a1df1a69563cc10b columnFamilyName B 2024-11-17T22:40:22,133 DEBUG [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:22,134 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(327): Store=3893ec8e69968d25a1df1a69563cc10b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:40:22,134 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,135 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3893ec8e69968d25a1df1a69563cc10b columnFamilyName C 2024-11-17T22:40:22,135 DEBUG [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:22,135 INFO [StoreOpener-3893ec8e69968d25a1df1a69563cc10b-1 {}] regionserver.HStore(327): Store=3893ec8e69968d25a1df1a69563cc10b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:40:22,135 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,136 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,137 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,138 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,138 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,138 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T22:40:22,140 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,141 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 3893ec8e69968d25a1df1a69563cc10b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73566446, jitterRate=0.09622547030448914}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T22:40:22,141 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,141 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 3893ec8e69968d25a1df1a69563cc10b: Running coprocessor pre-open hook at 1731883222119Writing region info on filesystem at 1731883222119Initializing all the Stores at 1731883222120 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883222120Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883222120Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883222120Cleaning up temporary data from old regions at 1731883222138 (+18 ms)Running coprocessor post-open hooks at 1731883222141 (+3 ms)Region opened successfully at 1731883222141 2024-11-17T22:40:22,142 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., pid=43, masterSystemTime=1731883222115 2024-11-17T22:40:22,144 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,144 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,144 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=3893ec8e69968d25a1df1a69563cc10b, regionState=OPEN, openSeqNum=5, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:40:22,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-17T22:40:22,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 in 184 msec 2024-11-17T22:40:22,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-17T22:40:22,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, REOPEN/MOVE in 508 msec 2024-11-17T22:40:22,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-17T22:40:22,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 519 msec 2024-11-17T22:40:22,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 948 msec 2024-11-17T22:40:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-17T22:40:22,158 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:22,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:22,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:22,159 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:22,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@177239ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:22,161 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:22,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c4809f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:22,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:22,163 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:22,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:22,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ed4814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:22,165 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22dd079f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:22,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:22,167 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a55037, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:22,169 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bce73a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:22,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:22,171 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79a51495, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:22,173 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d8cc9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,174 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:22,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:22,175 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@727a01ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:22,177 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:22,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1d8a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,179 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:22,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:22,180 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a2e3de2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:22,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:22,182 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@352607d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:22,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:22,184 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e506cbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:22,185 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d0f42b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:22,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:22,187 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:22,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:22,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:22,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4829f175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:22,188 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51798ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:22,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:22,191 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598a92fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:22,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f33b51f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:22,193 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:22,193 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:22,193 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:22,194 DEBUG [Thread-631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:22,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:22,194 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:22,195 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:22,195 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:22,196 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:22,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:22,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:22,197 DEBUG [Thread-618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:22,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:22,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:22,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:22,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:22,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-17T22:40:22,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T22:40:22,199 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:22,200 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:22,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:22,201 DEBUG [Thread-629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:22,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:22,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:40:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:40:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:40:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:40:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:40:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:40:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:40:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:40:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:22,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:40:22,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172672187955f343e6a9f11228e8dbb804_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883222219/Put/seqid=0 2024-11-17T22:40:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T22:40:22,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741947_1123 (size=12154) 2024-11-17T22:40:22,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883282299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883282299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883282302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883282302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883282304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T22:40:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883282417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883282417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883282417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883282418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883282428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T22:40:22,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:22,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T22:40:22,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883282627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883282637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883282638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883282638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883282637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T22:40:22,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:22,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,711 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:22,716 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172672187955f343e6a9f11228e8dbb804_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172672187955f343e6a9f11228e8dbb804_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:22,719 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/5a34bbb1f18840b3990e1d6b9c878a35, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:22,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/5a34bbb1f18840b3990e1d6b9c878a35 is 175, key is test_row_0/A:col10/1731883222219/Put/seqid=0 2024-11-17T22:40:22,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741948_1124 (size=30955) 2024-11-17T22:40:22,741 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/5a34bbb1f18840b3990e1d6b9c878a35 2024-11-17T22:40:22,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/68559556f69040429bef6be4cbab5812 is 50, key is test_row_0/B:col10/1731883222219/Put/seqid=0 2024-11-17T22:40:22,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T22:40:22,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:22,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T22:40:22,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741949_1125 (size=12001) 2024-11-17T22:40:22,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/68559556f69040429bef6be4cbab5812 2024-11-17T22:40:22,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/3ce03cf6494f4dff937c83ad56c8b173 is 50, key is test_row_0/C:col10/1731883222219/Put/seqid=0 2024-11-17T22:40:22,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883282937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883282947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883282947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883282948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883282948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:22,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741950_1126 (size=12001) 2024-11-17T22:40:22,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/3ce03cf6494f4dff937c83ad56c8b173 2024-11-17T22:40:22,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/5a34bbb1f18840b3990e1d6b9c878a35 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/5a34bbb1f18840b3990e1d6b9c878a35 2024-11-17T22:40:22,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T22:40:22,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:22,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:22,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:22,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/5a34bbb1f18840b3990e1d6b9c878a35, entries=150, sequenceid=16, filesize=30.2 K 2024-11-17T22:40:22,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/68559556f69040429bef6be4cbab5812 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/68559556f69040429bef6be4cbab5812 2024-11-17T22:40:22,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/68559556f69040429bef6be4cbab5812, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T22:40:22,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/3ce03cf6494f4dff937c83ad56c8b173 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3ce03cf6494f4dff937c83ad56c8b173 2024-11-17T22:40:23,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3ce03cf6494f4dff937c83ad56c8b173, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T22:40:23,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3893ec8e69968d25a1df1a69563cc10b in 788ms, sequenceid=16, compaction requested=false 2024-11-17T22:40:23,008 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T22:40:23,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:23,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T22:40:23,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:23,128 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:40:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117de1083874cd84c85a58533ede738bfef_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883222302/Put/seqid=0 2024-11-17T22:40:23,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741951_1127 (size=12154) 2024-11-17T22:40:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T22:40:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:23,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:23,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883283475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883283475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883283477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883283477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883283479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883283587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883283588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883283588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883283588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883283588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:23,617 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117de1083874cd84c85a58533ede738bfef_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117de1083874cd84c85a58533ede738bfef_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/be9d15cc8e2f4f2ab6f396e188b5d6a6, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/be9d15cc8e2f4f2ab6f396e188b5d6a6 is 175, key is test_row_0/A:col10/1731883222302/Put/seqid=0 2024-11-17T22:40:23,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741952_1128 (size=30955) 2024-11-17T22:40:23,633 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/be9d15cc8e2f4f2ab6f396e188b5d6a6 2024-11-17T22:40:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/c375e18eeaaa4fefbdf0803acb4245dd is 50, key is test_row_0/B:col10/1731883222302/Put/seqid=0 2024-11-17T22:40:23,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741953_1129 (size=12001) 2024-11-17T22:40:23,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883283797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883283797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883283797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883283798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883283798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:23,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:23,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:23,985 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T22:40:24,061 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/c375e18eeaaa4fefbdf0803acb4245dd 2024-11-17T22:40:24,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/2c79c992fda54e8ea5a8dbc0f2cbf296 is 50, key is test_row_0/C:col10/1731883222302/Put/seqid=0 2024-11-17T22:40:24,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741954_1130 (size=12001) 2024-11-17T22:40:24,093 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/2c79c992fda54e8ea5a8dbc0f2cbf296 2024-11-17T22:40:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/be9d15cc8e2f4f2ab6f396e188b5d6a6 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/be9d15cc8e2f4f2ab6f396e188b5d6a6 2024-11-17T22:40:24,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883284107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883284117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,123 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/be9d15cc8e2f4f2ab6f396e188b5d6a6, entries=150, sequenceid=40, filesize=30.2 K 2024-11-17T22:40:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883284117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883284117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883284117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/c375e18eeaaa4fefbdf0803acb4245dd as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c375e18eeaaa4fefbdf0803acb4245dd 2024-11-17T22:40:24,144 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c375e18eeaaa4fefbdf0803acb4245dd, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T22:40:24,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/2c79c992fda54e8ea5a8dbc0f2cbf296 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2c79c992fda54e8ea5a8dbc0f2cbf296 2024-11-17T22:40:24,155 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2c79c992fda54e8ea5a8dbc0f2cbf296, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T22:40:24,157 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3893ec8e69968d25a1df1a69563cc10b in 1028ms, sequenceid=40, compaction requested=false 2024-11-17T22:40:24,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:24,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:24,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-17T22:40:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-17T22:40:24,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-17T22:40:24,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9580 sec 2024-11-17T22:40:24,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.9660 sec 2024-11-17T22:40:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T22:40:24,347 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:24,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-17T22:40:24,350 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:24,350 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:24,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T22:40:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T22:40:24,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T22:40:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:24,505 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:40:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111792f8e4fc3a9f4767ae25c50b852fec96_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883223473/Put/seqid=0 2024-11-17T22:40:24,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741955_1131 (size=12154) 2024-11-17T22:40:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:24,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T22:40:24,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883284682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883284683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883284684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883284688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883284687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883284797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883284797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883284797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883284807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883284807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:24,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:24,942 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111792f8e4fc3a9f4767ae25c50b852fec96_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111792f8e4fc3a9f4767ae25c50b852fec96_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/9c16adc132c2443c9966f473b31af196, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:24,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/9c16adc132c2443c9966f473b31af196 is 175, key is test_row_0/A:col10/1731883223473/Put/seqid=0 2024-11-17T22:40:24,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741956_1132 (size=30955) 2024-11-17T22:40:24,972 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/9c16adc132c2443c9966f473b31af196 2024-11-17T22:40:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T22:40:24,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/78ff77f9fc50462fb2b798f5eedee12e is 50, key is test_row_0/B:col10/1731883223473/Put/seqid=0 2024-11-17T22:40:24,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741957_1133 (size=12001) 2024-11-17T22:40:25,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883285007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883285014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883285014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883285017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883285018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883285317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883285327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883285328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883285328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883285328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,397 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/78ff77f9fc50462fb2b798f5eedee12e 2024-11-17T22:40:25,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/ffbf715b844f4447b760752e4d27a182 is 50, key is test_row_0/C:col10/1731883223473/Put/seqid=0 2024-11-17T22:40:25,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741958_1134 (size=12001) 2024-11-17T22:40:25,409 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/ffbf715b844f4447b760752e4d27a182 2024-11-17T22:40:25,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/9c16adc132c2443c9966f473b31af196 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9c16adc132c2443c9966f473b31af196 2024-11-17T22:40:25,419 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9c16adc132c2443c9966f473b31af196, entries=150, sequenceid=52, filesize=30.2 K 2024-11-17T22:40:25,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/78ff77f9fc50462fb2b798f5eedee12e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/78ff77f9fc50462fb2b798f5eedee12e 2024-11-17T22:40:25,425 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/78ff77f9fc50462fb2b798f5eedee12e, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T22:40:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/ffbf715b844f4447b760752e4d27a182 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ffbf715b844f4447b760752e4d27a182 2024-11-17T22:40:25,429 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ffbf715b844f4447b760752e4d27a182, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T22:40:25,430 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3893ec8e69968d25a1df1a69563cc10b in 925ms, sequenceid=52, compaction requested=true 2024-11-17T22:40:25,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:25,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:25,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-17T22:40:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-17T22:40:25,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-17T22:40:25,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0820 sec 2024-11-17T22:40:25,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.0860 sec 2024-11-17T22:40:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T22:40:25,487 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:25,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-17T22:40:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T22:40:25,489 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:25,490 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:25,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T22:40:25,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T22:40:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:25,642 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:40:25,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d3020e49991e401dacd900d12e0f1b1e_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883224666/Put/seqid=0 2024-11-17T22:40:25,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741959_1135 (size=12154) 2024-11-17T22:40:25,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:25,663 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d3020e49991e401dacd900d12e0f1b1e_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d3020e49991e401dacd900d12e0f1b1e_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:25,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/2e7ab5ad23a54136a365ae37730c673c, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:25,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/2e7ab5ad23a54136a365ae37730c673c is 175, key is test_row_0/A:col10/1731883224666/Put/seqid=0 2024-11-17T22:40:25,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741960_1136 (size=30955) 2024-11-17T22:40:25,683 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/2e7ab5ad23a54136a365ae37730c673c 2024-11-17T22:40:25,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/600eab9e4b464b21bc4cd61156884f41 is 50, key is test_row_0/B:col10/1731883224666/Put/seqid=0 2024-11-17T22:40:25,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741961_1137 (size=12001) 2024-11-17T22:40:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T22:40:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:25,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:25,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883285842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883285847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883285847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883285848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883285848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883285957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883285957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883285957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883285957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883285957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:26,110 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/600eab9e4b464b21bc4cd61156884f41 2024-11-17T22:40:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T22:40:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/3371461477d2461ebebf525ccf10f20b is 50, key is test_row_0/C:col10/1731883224666/Put/seqid=0 2024-11-17T22:40:26,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741962_1138 (size=12001) 2024-11-17T22:40:26,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883286167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:26,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883286167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883286167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883286167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883286167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:26,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883286477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:26,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883286478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883286478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:26,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883286478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883286478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:26,535 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/3371461477d2461ebebf525ccf10f20b 2024-11-17T22:40:26,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/2e7ab5ad23a54136a365ae37730c673c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/2e7ab5ad23a54136a365ae37730c673c 2024-11-17T22:40:26,552 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/2e7ab5ad23a54136a365ae37730c673c, entries=150, sequenceid=76, filesize=30.2 K 2024-11-17T22:40:26,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/600eab9e4b464b21bc4cd61156884f41 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/600eab9e4b464b21bc4cd61156884f41 2024-11-17T22:40:26,567 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/600eab9e4b464b21bc4cd61156884f41, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T22:40:26,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/3371461477d2461ebebf525ccf10f20b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3371461477d2461ebebf525ccf10f20b 2024-11-17T22:40:26,582 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3371461477d2461ebebf525ccf10f20b, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T22:40:26,585 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3893ec8e69968d25a1df1a69563cc10b in 942ms, sequenceid=76, compaction requested=true 2024-11-17T22:40:26,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:26,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:26,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-17T22:40:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-17T22:40:26,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-17T22:40:26,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0960 sec 2024-11-17T22:40:26,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.1050 sec 2024-11-17T22:40:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T22:40:26,627 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:26,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-17T22:40:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T22:40:26,630 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:26,631 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:26,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T22:40:26,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T22:40:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:26,788 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:40:26,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ae85028ab17c447bb787e9e0f0164d83_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883225846/Put/seqid=0 2024-11-17T22:40:26,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741963_1139 (size=12154) 2024-11-17T22:40:26,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:26,898 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ae85028ab17c447bb787e9e0f0164d83_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ae85028ab17c447bb787e9e0f0164d83_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:26,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/a379540fe4264c61b0d027cfd8badae0, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:26,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/a379540fe4264c61b0d027cfd8badae0 is 175, key is test_row_0/A:col10/1731883225846/Put/seqid=0 2024-11-17T22:40:26,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741964_1140 (size=30955) 2024-11-17T22:40:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T22:40:26,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:27,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883287034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883287036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883287037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883287039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883287044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883287159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883287159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883287159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883287159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883287160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T22:40:27,316 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/a379540fe4264c61b0d027cfd8badae0 2024-11-17T22:40:27,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/43a1a6296668460f88759dc5af8c77b4 is 50, key is test_row_0/B:col10/1731883225846/Put/seqid=0 2024-11-17T22:40:27,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741965_1141 (size=12001) 2024-11-17T22:40:27,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883287377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883287377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883287377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883287377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883287377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883287688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883287688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883287688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883287688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883287688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:27,741 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/43a1a6296668460f88759dc5af8c77b4 2024-11-17T22:40:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/a76059af63864384bd37a425f32d8f7c is 50, key is test_row_0/C:col10/1731883225846/Put/seqid=0 2024-11-17T22:40:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T22:40:27,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741966_1142 (size=12001) 2024-11-17T22:40:28,194 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/a76059af63864384bd37a425f32d8f7c 2024-11-17T22:40:28,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/a379540fe4264c61b0d027cfd8badae0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/a379540fe4264c61b0d027cfd8badae0 2024-11-17T22:40:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883288197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883288198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:28,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883288198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:28,208 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/a379540fe4264c61b0d027cfd8badae0, entries=150, sequenceid=88, filesize=30.2 K 2024-11-17T22:40:28,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/43a1a6296668460f88759dc5af8c77b4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/43a1a6296668460f88759dc5af8c77b4 2024-11-17T22:40:28,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883288207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:28,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883288207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:28,219 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/43a1a6296668460f88759dc5af8c77b4, entries=150, sequenceid=88, filesize=11.7 K 2024-11-17T22:40:28,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/a76059af63864384bd37a425f32d8f7c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a76059af63864384bd37a425f32d8f7c 2024-11-17T22:40:28,229 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a76059af63864384bd37a425f32d8f7c, entries=150, sequenceid=88, filesize=11.7 K 2024-11-17T22:40:28,230 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3893ec8e69968d25a1df1a69563cc10b in 1442ms, sequenceid=88, compaction requested=true 2024-11-17T22:40:28,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:28,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:28,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-17T22:40:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-17T22:40:28,236 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-17T22:40:28,236 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6010 sec 2024-11-17T22:40:28,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.6090 sec 2024-11-17T22:40:28,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T22:40:28,618 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T22:40:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T22:40:28,777 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:28,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-17T22:40:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T22:40:28,782 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:28,784 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:28,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T22:40:28,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T22:40:28,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:28,938 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:40:28,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111720de093bd2a74b4088a43f78b6a662a6_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883227037/Put/seqid=0 2024-11-17T22:40:28,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741967_1143 (size=12154) 2024-11-17T22:40:28,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:28,958 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111720de093bd2a74b4088a43f78b6a662a6_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111720de093bd2a74b4088a43f78b6a662a6_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:28,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/b2619a91ecbd4f94b2f2119d76d1d2e8, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:28,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/b2619a91ecbd4f94b2f2119d76d1d2e8 is 175, key is test_row_0/A:col10/1731883227037/Put/seqid=0 2024-11-17T22:40:28,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741968_1144 (size=30955) 2024-11-17T22:40:28,971 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/b2619a91ecbd4f94b2f2119d76d1d2e8 2024-11-17T22:40:28,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/70ebb6d4a08e4eb5be999e91d3f44a11 is 50, key is test_row_0/B:col10/1731883227037/Put/seqid=0 2024-11-17T22:40:29,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741969_1145 (size=12001) 2024-11-17T22:40:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T22:40:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:29,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:29,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883289230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883289230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883289231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883289232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883289233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883289347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883289347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883289347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883289347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883289347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,403 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/70ebb6d4a08e4eb5be999e91d3f44a11 2024-11-17T22:40:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T22:40:29,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/54da967fcadf41b89626ad30aab23ad2 is 50, key is test_row_0/C:col10/1731883227037/Put/seqid=0 2024-11-17T22:40:29,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741970_1146 (size=12001) 2024-11-17T22:40:29,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883289557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883289557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883289557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883289567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883289567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,831 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/54da967fcadf41b89626ad30aab23ad2 2024-11-17T22:40:29,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/b2619a91ecbd4f94b2f2119d76d1d2e8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/b2619a91ecbd4f94b2f2119d76d1d2e8 2024-11-17T22:40:29,841 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/b2619a91ecbd4f94b2f2119d76d1d2e8, entries=150, sequenceid=112, filesize=30.2 K 2024-11-17T22:40:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/70ebb6d4a08e4eb5be999e91d3f44a11 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/70ebb6d4a08e4eb5be999e91d3f44a11 2024-11-17T22:40:29,848 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/70ebb6d4a08e4eb5be999e91d3f44a11, entries=150, sequenceid=112, filesize=11.7 K 2024-11-17T22:40:29,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/54da967fcadf41b89626ad30aab23ad2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/54da967fcadf41b89626ad30aab23ad2 2024-11-17T22:40:29,857 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/54da967fcadf41b89626ad30aab23ad2, entries=150, sequenceid=112, filesize=11.7 K 2024-11-17T22:40:29,859 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3893ec8e69968d25a1df1a69563cc10b in 921ms, sequenceid=112, compaction requested=true 2024-11-17T22:40:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-17T22:40:29,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-17T22:40:29,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-17T22:40:29,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0760 sec 2024-11-17T22:40:29,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.0850 sec 2024-11-17T22:40:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:29,870 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:29,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:40:29,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172e252870b8a247dd9632175ee25008f8_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883229232/Put/seqid=0 2024-11-17T22:40:29,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741971_1147 (size=14594) 2024-11-17T22:40:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T22:40:29,917 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:29,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-17T22:40:29,921 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T22:40:29,923 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:29,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:29,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883289922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883289924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883289924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883289928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:29,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883289928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T22:40:30,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883290047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883290047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883290047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883290047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883290047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T22:40:30,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:30,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:30,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:30,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:30,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:30,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T22:40:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:30,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T22:40:30,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883290257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883290258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883290258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883290258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883290258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,307 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:30,311 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172e252870b8a247dd9632175ee25008f8_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e252870b8a247dd9632175ee25008f8_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:30,313 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/09ace82e77a748109d53c857b9b573e9, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:30,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/09ace82e77a748109d53c857b9b573e9 is 175, key is test_row_0/A:col10/1731883229232/Put/seqid=0 2024-11-17T22:40:30,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741972_1148 (size=39549) 2024-11-17T22:40:30,331 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=125, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/09ace82e77a748109d53c857b9b573e9 2024-11-17T22:40:30,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/0e4058ba403b4cb58aaaf5f37f2b527b is 50, key is test_row_0/B:col10/1731883229232/Put/seqid=0 2024-11-17T22:40:30,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741973_1149 (size=12001) 2024-11-17T22:40:30,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/0e4058ba403b4cb58aaaf5f37f2b527b 2024-11-17T22:40:30,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/ec7427fd49a443dfb71325ce691cd52f is 50, key is test_row_0/C:col10/1731883229232/Put/seqid=0 2024-11-17T22:40:30,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741974_1150 (size=12001) 2024-11-17T22:40:30,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/ec7427fd49a443dfb71325ce691cd52f 2024-11-17T22:40:30,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/09ace82e77a748109d53c857b9b573e9 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/09ace82e77a748109d53c857b9b573e9 2024-11-17T22:40:30,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T22:40:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:30,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:30,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:30,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:30,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/09ace82e77a748109d53c857b9b573e9, entries=200, sequenceid=125, filesize=38.6 K 2024-11-17T22:40:30,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/0e4058ba403b4cb58aaaf5f37f2b527b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/0e4058ba403b4cb58aaaf5f37f2b527b 2024-11-17T22:40:30,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/0e4058ba403b4cb58aaaf5f37f2b527b, entries=150, sequenceid=125, filesize=11.7 K 2024-11-17T22:40:30,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/ec7427fd49a443dfb71325ce691cd52f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ec7427fd49a443dfb71325ce691cd52f 2024-11-17T22:40:30,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ec7427fd49a443dfb71325ce691cd52f, entries=150, sequenceid=125, filesize=11.7 K 2024-11-17T22:40:30,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3893ec8e69968d25a1df1a69563cc10b in 537ms, sequenceid=125, compaction requested=true 2024-11-17T22:40:30,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:30,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:30,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:30,408 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-17T22:40:30,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:30,409 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-17T22:40:30,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:30,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:30,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:30,414 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84007 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-17T22:40:30,414 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/B is initiating minor compaction (all files) 2024-11-17T22:40:30,414 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/B in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:30,414 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/68559556f69040429bef6be4cbab5812, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c375e18eeaaa4fefbdf0803acb4245dd, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/78ff77f9fc50462fb2b798f5eedee12e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/600eab9e4b464b21bc4cd61156884f41, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/43a1a6296668460f88759dc5af8c77b4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/70ebb6d4a08e4eb5be999e91d3f44a11, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/0e4058ba403b4cb58aaaf5f37f2b527b] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=82.0 K 2024-11-17T22:40:30,414 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 225279 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-17T22:40:30,414 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/A is initiating minor compaction (all files) 2024-11-17T22:40:30,415 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/A in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:30,415 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/5a34bbb1f18840b3990e1d6b9c878a35, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/be9d15cc8e2f4f2ab6f396e188b5d6a6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9c16adc132c2443c9966f473b31af196, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/2e7ab5ad23a54136a365ae37730c673c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/a379540fe4264c61b0d027cfd8badae0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/b2619a91ecbd4f94b2f2119d76d1d2e8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/09ace82e77a748109d53c857b9b573e9] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=220.0 K 2024-11-17T22:40:30,415 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:30,415 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/5a34bbb1f18840b3990e1d6b9c878a35, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/be9d15cc8e2f4f2ab6f396e188b5d6a6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9c16adc132c2443c9966f473b31af196, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/2e7ab5ad23a54136a365ae37730c673c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/a379540fe4264c61b0d027cfd8badae0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/b2619a91ecbd4f94b2f2119d76d1d2e8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/09ace82e77a748109d53c857b9b573e9] 2024-11-17T22:40:30,416 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 68559556f69040429bef6be4cbab5812, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731883222217 2024-11-17T22:40:30,417 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a34bbb1f18840b3990e1d6b9c878a35, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731883222217 2024-11-17T22:40:30,417 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting c375e18eeaaa4fefbdf0803acb4245dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731883222290 2024-11-17T22:40:30,417 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting be9d15cc8e2f4f2ab6f396e188b5d6a6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731883222290 2024-11-17T22:40:30,417 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 78ff77f9fc50462fb2b798f5eedee12e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731883223472 2024-11-17T22:40:30,417 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 600eab9e4b464b21bc4cd61156884f41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731883224666 2024-11-17T22:40:30,417 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c16adc132c2443c9966f473b31af196, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731883223472 2024-11-17T22:40:30,418 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 43a1a6296668460f88759dc5af8c77b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731883225842 2024-11-17T22:40:30,418 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e7ab5ad23a54136a365ae37730c673c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731883224666 2024-11-17T22:40:30,418 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 70ebb6d4a08e4eb5be999e91d3f44a11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731883227021 2024-11-17T22:40:30,418 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting a379540fe4264c61b0d027cfd8badae0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731883225842 2024-11-17T22:40:30,418 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e4058ba403b4cb58aaaf5f37f2b527b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731883229231 2024-11-17T22:40:30,419 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2619a91ecbd4f94b2f2119d76d1d2e8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731883227021 2024-11-17T22:40:30,419 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09ace82e77a748109d53c857b9b573e9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731883229230 2024-11-17T22:40:30,447 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#B#compaction#128 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:30,448 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/80cd30ba39014427b061225ad78acdca is 50, key is test_row_0/B:col10/1731883229232/Put/seqid=0 2024-11-17T22:40:30,450 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:30,460 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117bc20ac69ba8a4f5fad2df8ccadfb874f_3893ec8e69968d25a1df1a69563cc10b store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:30,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741975_1151 (size=12241) 2024-11-17T22:40:30,467 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117bc20ac69ba8a4f5fad2df8ccadfb874f_3893ec8e69968d25a1df1a69563cc10b, store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:30,467 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bc20ac69ba8a4f5fad2df8ccadfb874f_3893ec8e69968d25a1df1a69563cc10b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:30,478 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/80cd30ba39014427b061225ad78acdca as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/80cd30ba39014427b061225ad78acdca 2024-11-17T22:40:30,487 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/B of 3893ec8e69968d25a1df1a69563cc10b into 80cd30ba39014427b061225ad78acdca(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:30,487 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:30,487 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/B, priority=9, startTime=1731883230408; duration=0sec 2024-11-17T22:40:30,487 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:30,487 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:B 2024-11-17T22:40:30,487 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-17T22:40:30,494 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84007 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-17T22:40:30,494 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/C is initiating minor compaction (all files) 2024-11-17T22:40:30,494 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/C in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:30,494 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3ce03cf6494f4dff937c83ad56c8b173, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2c79c992fda54e8ea5a8dbc0f2cbf296, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ffbf715b844f4447b760752e4d27a182, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3371461477d2461ebebf525ccf10f20b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a76059af63864384bd37a425f32d8f7c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/54da967fcadf41b89626ad30aab23ad2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ec7427fd49a443dfb71325ce691cd52f] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=82.0 K 2024-11-17T22:40:30,495 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ce03cf6494f4dff937c83ad56c8b173, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731883222217 2024-11-17T22:40:30,495 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c79c992fda54e8ea5a8dbc0f2cbf296, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731883222290 2024-11-17T22:40:30,496 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting ffbf715b844f4447b760752e4d27a182, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731883223472 2024-11-17T22:40:30,496 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 3371461477d2461ebebf525ccf10f20b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731883224666 2024-11-17T22:40:30,496 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting a76059af63864384bd37a425f32d8f7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731883225842 2024-11-17T22:40:30,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741976_1152 (size=4469) 2024-11-17T22:40:30,500 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 54da967fcadf41b89626ad30aab23ad2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731883227021 2024-11-17T22:40:30,502 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting ec7427fd49a443dfb71325ce691cd52f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731883229231 2024-11-17T22:40:30,506 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#A#compaction#129 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:30,507 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/dcb9a0481c664f41a30ddf6dd6b81c7e is 175, key is test_row_0/A:col10/1731883229232/Put/seqid=0 2024-11-17T22:40:30,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741977_1153 (size=31195) 2024-11-17T22:40:30,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T22:40:30,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:30,539 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T22:40:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T22:40:30,548 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/dcb9a0481c664f41a30ddf6dd6b81c7e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/dcb9a0481c664f41a30ddf6dd6b81c7e 2024-11-17T22:40:30,551 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#C#compaction#130 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:30,552 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/5d3daed4982b410e989b82783789a3d7 is 50, key is test_row_0/C:col10/1731883229232/Put/seqid=0 2024-11-17T22:40:30,556 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/A of 3893ec8e69968d25a1df1a69563cc10b into dcb9a0481c664f41a30ddf6dd6b81c7e(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:30,556 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:30,556 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/A, priority=9, startTime=1731883230408; duration=0sec 2024-11-17T22:40:30,556 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:30,556 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:A 2024-11-17T22:40:30,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b9eb65d0091d4c6fa5baf482effda9c8_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883229924/Put/seqid=0 2024-11-17T22:40:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:30,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:30,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741978_1154 (size=12241) 2024-11-17T22:40:30,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883290583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,590 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/5d3daed4982b410e989b82783789a3d7 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/5d3daed4982b410e989b82783789a3d7 2024-11-17T22:40:30,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883290583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883290584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883290585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741979_1155 (size=12304) 2024-11-17T22:40:30,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:30,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883290588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,601 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/C of 3893ec8e69968d25a1df1a69563cc10b into 5d3daed4982b410e989b82783789a3d7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:30,601 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:30,601 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/C, priority=9, startTime=1731883230409; duration=0sec 2024-11-17T22:40:30,602 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:30,602 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:C 2024-11-17T22:40:30,605 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b9eb65d0091d4c6fa5baf482effda9c8_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b9eb65d0091d4c6fa5baf482effda9c8_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:30,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/abcbfc24611945ae85900ecdc32d46fa, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:30,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/abcbfc24611945ae85900ecdc32d46fa is 175, key is test_row_0/A:col10/1731883229924/Put/seqid=0 2024-11-17T22:40:30,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741980_1156 (size=31105) 2024-11-17T22:40:30,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883290697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883290697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883290699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883290699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883290707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883290907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883290907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883290907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883290907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:30,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883290917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:31,024 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/abcbfc24611945ae85900ecdc32d46fa 2024-11-17T22:40:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T22:40:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/d3f512fcaf1d4d3f95768c2513a18396 is 50, key is test_row_0/B:col10/1731883229924/Put/seqid=0 2024-11-17T22:40:31,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741981_1157 (size=12151) 2024-11-17T22:40:31,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883291217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:31,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:31,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883291217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:31,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883291218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:31,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883291218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:31,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883291227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:31,510 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/d3f512fcaf1d4d3f95768c2513a18396 2024-11-17T22:40:31,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/abc7729aeb944ae69bbf09b6f52ede38 is 50, key is test_row_0/C:col10/1731883229924/Put/seqid=0 2024-11-17T22:40:31,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741982_1158 (size=12151) 2024-11-17T22:40:31,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883291727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:31,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:31,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883291737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883291737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:31,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883291737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:31,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883291746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:31,947 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/abc7729aeb944ae69bbf09b6f52ede38 2024-11-17T22:40:31,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/abcbfc24611945ae85900ecdc32d46fa as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/abcbfc24611945ae85900ecdc32d46fa 2024-11-17T22:40:31,957 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/abcbfc24611945ae85900ecdc32d46fa, entries=150, sequenceid=149, filesize=30.4 K 2024-11-17T22:40:31,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/d3f512fcaf1d4d3f95768c2513a18396 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/d3f512fcaf1d4d3f95768c2513a18396 2024-11-17T22:40:31,963 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/d3f512fcaf1d4d3f95768c2513a18396, entries=150, sequenceid=149, filesize=11.9 K 2024-11-17T22:40:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/abc7729aeb944ae69bbf09b6f52ede38 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/abc7729aeb944ae69bbf09b6f52ede38 2024-11-17T22:40:31,968 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/abc7729aeb944ae69bbf09b6f52ede38, entries=150, sequenceid=149, filesize=11.9 K 2024-11-17T22:40:31,970 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 3893ec8e69968d25a1df1a69563cc10b in 1431ms, sequenceid=149, compaction requested=false 2024-11-17T22:40:31,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:31,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:31,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-17T22:40:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-17T22:40:31,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-17T22:40:31,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0480 sec 2024-11-17T22:40:31,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.0570 sec 2024-11-17T22:40:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T22:40:32,067 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:32,068 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-17T22:40:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T22:40:32,070 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:32,070 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:32,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T22:40:32,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T22:40:32,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:32,223 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:40:32,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e693aa50dbaa4ca28e7a772251f0f163_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883230582/Put/seqid=0 2024-11-17T22:40:32,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741983_1159 (size=12304) 2024-11-17T22:40:32,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:32,276 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e693aa50dbaa4ca28e7a772251f0f163_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e693aa50dbaa4ca28e7a772251f0f163_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:32,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/f6c673f33f5243d5af1a2123ab7658b2, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:32,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/f6c673f33f5243d5af1a2123ab7658b2 is 175, key is test_row_0/A:col10/1731883230582/Put/seqid=0 2024-11-17T22:40:32,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741984_1160 (size=31105) 2024-11-17T22:40:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T22:40:32,552 INFO [master/1a6e40b21a48:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-17T22:40:32,552 INFO [master/1a6e40b21a48:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-17T22:40:32,691 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/f6c673f33f5243d5af1a2123ab7658b2 2024-11-17T22:40:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T22:40:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/7b25a39096b341f288add0f3e6078ad5 is 50, key is test_row_0/B:col10/1731883230582/Put/seqid=0 2024-11-17T22:40:32,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741985_1161 (size=12151) 2024-11-17T22:40:32,713 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/7b25a39096b341f288add0f3e6078ad5 2024-11-17T22:40:32,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/194c2603b3804b7eaff60f3c704281de is 50, key is test_row_0/C:col10/1731883230582/Put/seqid=0 2024-11-17T22:40:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:32,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:32,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741986_1162 (size=12151) 2024-11-17T22:40:32,760 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/194c2603b3804b7eaff60f3c704281de 2024-11-17T22:40:32,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/f6c673f33f5243d5af1a2123ab7658b2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/f6c673f33f5243d5af1a2123ab7658b2 2024-11-17T22:40:32,776 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/f6c673f33f5243d5af1a2123ab7658b2, entries=150, sequenceid=164, filesize=30.4 K 2024-11-17T22:40:32,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/7b25a39096b341f288add0f3e6078ad5 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/7b25a39096b341f288add0f3e6078ad5 2024-11-17T22:40:32,786 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/7b25a39096b341f288add0f3e6078ad5, entries=150, sequenceid=164, filesize=11.9 K 2024-11-17T22:40:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/194c2603b3804b7eaff60f3c704281de as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/194c2603b3804b7eaff60f3c704281de 2024-11-17T22:40:32,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883292784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:32,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:32,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883292786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:32,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:32,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883292791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:32,797 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/194c2603b3804b7eaff60f3c704281de, entries=150, sequenceid=164, filesize=11.9 K 2024-11-17T22:40:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883292793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:32,798 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3893ec8e69968d25a1df1a69563cc10b in 575ms, sequenceid=164, compaction requested=true 2024-11-17T22:40:32,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:32,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:32,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-17T22:40:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-17T22:40:32,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-17T22:40:32,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 729 msec 2024-11-17T22:40:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:32,804 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:32,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:40:32,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 735 msec 2024-11-17T22:40:32,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c22ac9542412464c8a7008cabd070bfe_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883232783/Put/seqid=0 2024-11-17T22:40:32,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741987_1163 (size=14794) 2024-11-17T22:40:32,862 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:32,867 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c22ac9542412464c8a7008cabd070bfe_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c22ac9542412464c8a7008cabd070bfe_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:32,868 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/d94d73d4122347608613af34da7b3614, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:32,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/d94d73d4122347608613af34da7b3614 is 175, key is test_row_0/A:col10/1731883232783/Put/seqid=0 2024-11-17T22:40:32,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883292875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:32,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741988_1164 (size=39749) 2024-11-17T22:40:32,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883292906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:32,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883292906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:32,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883292907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:32,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883292907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:32,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883292987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883293118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883293118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883293119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883293119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T22:40:33,208 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:33,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-17T22:40:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T22:40:33,211 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:33,212 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:33,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883293208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,285 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/d94d73d4122347608613af34da7b3614 2024-11-17T22:40:33,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/fb2e1338e892429b80eabdd0a36b78c3 is 50, key is test_row_0/B:col10/1731883232783/Put/seqid=0 2024-11-17T22:40:33,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T22:40:33,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741989_1165 (size=12151) 2024-11-17T22:40:33,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/fb2e1338e892429b80eabdd0a36b78c3 2024-11-17T22:40:33,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/d153f4cdf2064352a3edf11e2976e83d is 50, key is test_row_0/C:col10/1731883232783/Put/seqid=0 2024-11-17T22:40:33,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741990_1166 (size=12151) 2024-11-17T22:40:33,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/d153f4cdf2064352a3edf11e2976e83d 2024-11-17T22:40:33,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/d94d73d4122347608613af34da7b3614 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/d94d73d4122347608613af34da7b3614 2024-11-17T22:40:33,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/d94d73d4122347608613af34da7b3614, entries=200, sequenceid=188, filesize=38.8 K 2024-11-17T22:40:33,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/fb2e1338e892429b80eabdd0a36b78c3 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/fb2e1338e892429b80eabdd0a36b78c3 2024-11-17T22:40:33,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T22:40:33,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:33,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/fb2e1338e892429b80eabdd0a36b78c3, entries=150, sequenceid=188, filesize=11.9 K 2024-11-17T22:40:33,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/d153f4cdf2064352a3edf11e2976e83d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/d153f4cdf2064352a3edf11e2976e83d 2024-11-17T22:40:33,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/d153f4cdf2064352a3edf11e2976e83d, entries=150, sequenceid=188, filesize=11.9 K 2024-11-17T22:40:33,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3893ec8e69968d25a1df1a69563cc10b in 582ms, sequenceid=188, compaction requested=true 2024-11-17T22:40:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:33,386 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:33,386 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:33,392 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:33,392 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/B is initiating minor compaction (all files) 2024-11-17T22:40:33,392 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/B in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,392 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/80cd30ba39014427b061225ad78acdca, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/d3f512fcaf1d4d3f95768c2513a18396, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/7b25a39096b341f288add0f3e6078ad5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/fb2e1338e892429b80eabdd0a36b78c3] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=47.6 K 2024-11-17T22:40:33,393 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:33,393 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/A is initiating minor compaction (all files) 2024-11-17T22:40:33,394 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/A in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,394 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/dcb9a0481c664f41a30ddf6dd6b81c7e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/abcbfc24611945ae85900ecdc32d46fa, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/f6c673f33f5243d5af1a2123ab7658b2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/d94d73d4122347608613af34da7b3614] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=130.0 K 2024-11-17T22:40:33,394 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,394 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/dcb9a0481c664f41a30ddf6dd6b81c7e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/abcbfc24611945ae85900ecdc32d46fa, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/f6c673f33f5243d5af1a2123ab7658b2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/d94d73d4122347608613af34da7b3614] 2024-11-17T22:40:33,394 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 80cd30ba39014427b061225ad78acdca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731883229231 2024-11-17T22:40:33,396 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcb9a0481c664f41a30ddf6dd6b81c7e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731883229231 2024-11-17T22:40:33,396 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d3f512fcaf1d4d3f95768c2513a18396, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731883229921 2024-11-17T22:40:33,397 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting abcbfc24611945ae85900ecdc32d46fa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731883229921 2024-11-17T22:40:33,398 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b25a39096b341f288add0f3e6078ad5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731883230582 2024-11-17T22:40:33,398 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting fb2e1338e892429b80eabdd0a36b78c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731883232783 2024-11-17T22:40:33,399 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6c673f33f5243d5af1a2123ab7658b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731883230582 2024-11-17T22:40:33,400 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting d94d73d4122347608613af34da7b3614, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731883232783 2024-11-17T22:40:33,415 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#B#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:33,416 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/9d8d7f704a9d4e3d946d0773197bc7c5 is 50, key is test_row_0/B:col10/1731883232783/Put/seqid=0 2024-11-17T22:40:33,419 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:33,430 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:33,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:40:33,438 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117fe35f24d3c6946d9a7bd0415bc20690e_3893ec8e69968d25a1df1a69563cc10b store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:33,443 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117fe35f24d3c6946d9a7bd0415bc20690e_3893ec8e69968d25a1df1a69563cc10b, store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:33,443 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fe35f24d3c6946d9a7bd0415bc20690e_3893ec8e69968d25a1df1a69563cc10b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:33,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741991_1167 (size=12527) 2024-11-17T22:40:33,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117246eb50ac353415baa579d142d02f289_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883233429/Put/seqid=0 2024-11-17T22:40:33,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741992_1168 (size=4469) 2024-11-17T22:40:33,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741993_1169 (size=12304) 2024-11-17T22:40:33,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883293468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883293470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883293470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883293478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T22:40:33,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883293517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:33,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T22:40:33,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883293587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883293587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883293587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883293587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T22:40:33,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:33,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883293797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883293797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883293797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883293797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:33,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T22:40:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T22:40:33,856 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/9d8d7f704a9d4e3d946d0773197bc7c5 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/9d8d7f704a9d4e3d946d0773197bc7c5 2024-11-17T22:40:33,862 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/B of 3893ec8e69968d25a1df1a69563cc10b into 9d8d7f704a9d4e3d946d0773197bc7c5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:33,862 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:33,862 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/B, priority=12, startTime=1731883233386; duration=0sec 2024-11-17T22:40:33,862 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:33,862 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:B 2024-11-17T22:40:33,862 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:33,865 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#A#compaction#141 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:33,866 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/7e0dc59fa6e443a2bfdb67fb09e8a01e is 175, key is test_row_0/A:col10/1731883232783/Put/seqid=0 2024-11-17T22:40:33,869 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:33,869 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/C is initiating minor compaction (all files) 2024-11-17T22:40:33,869 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/C in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,869 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/5d3daed4982b410e989b82783789a3d7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/abc7729aeb944ae69bbf09b6f52ede38, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/194c2603b3804b7eaff60f3c704281de, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/d153f4cdf2064352a3edf11e2976e83d] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=47.6 K 2024-11-17T22:40:33,871 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d3daed4982b410e989b82783789a3d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731883229231 2024-11-17T22:40:33,876 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting abc7729aeb944ae69bbf09b6f52ede38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731883229921 2024-11-17T22:40:33,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:33,876 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 194c2603b3804b7eaff60f3c704281de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731883230582 2024-11-17T22:40:33,877 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d153f4cdf2064352a3edf11e2976e83d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731883232783 2024-11-17T22:40:33,881 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117246eb50ac353415baa579d142d02f289_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117246eb50ac353415baa579d142d02f289_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:33,882 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/9ef13b608dc04a9783d2782a1ea3b1c4, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:33,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/9ef13b608dc04a9783d2782a1ea3b1c4 is 175, key is test_row_0/A:col10/1731883233429/Put/seqid=0 2024-11-17T22:40:33,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741994_1170 (size=31481) 2024-11-17T22:40:33,903 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/7e0dc59fa6e443a2bfdb67fb09e8a01e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7e0dc59fa6e443a2bfdb67fb09e8a01e 2024-11-17T22:40:33,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741995_1171 (size=31105) 2024-11-17T22:40:33,911 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#C#compaction#143 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:33,911 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/2fa920313afc4096b289bd219d3664b3 is 50, key is test_row_0/C:col10/1731883232783/Put/seqid=0 2024-11-17T22:40:33,912 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/A of 3893ec8e69968d25a1df1a69563cc10b into 7e0dc59fa6e443a2bfdb67fb09e8a01e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:33,912 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:33,912 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/A, priority=12, startTime=1731883233386; duration=0sec 2024-11-17T22:40:33,912 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:33,913 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:A 2024-11-17T22:40:33,913 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/9ef13b608dc04a9783d2782a1ea3b1c4 2024-11-17T22:40:33,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741996_1172 (size=12527) 2024-11-17T22:40:33,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/db4bb70747cf4f22986205fbe5c90763 is 50, key is test_row_0/B:col10/1731883233429/Put/seqid=0 2024-11-17T22:40:33,934 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/2fa920313afc4096b289bd219d3664b3 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2fa920313afc4096b289bd219d3664b3 2024-11-17T22:40:33,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741997_1173 (size=12151) 2024-11-17T22:40:33,944 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/C of 3893ec8e69968d25a1df1a69563cc10b into 2fa920313afc4096b289bd219d3664b3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:33,945 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:33,945 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/C, priority=12, startTime=1731883233386; duration=0sec 2024-11-17T22:40:33,945 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:33,945 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:C 2024-11-17T22:40:33,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T22:40:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:33,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883294027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:34,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883294107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:34,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:34,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883294107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:34,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:34,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883294111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:34,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883294111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:34,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T22:40:34,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:34,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:34,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:34,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T22:40:34,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:34,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:34,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:34,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/db4bb70747cf4f22986205fbe5c90763 2024-11-17T22:40:34,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T22:40:34,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/c235c3a89d4e43e599a4dcf4367252b7 is 50, key is test_row_0/C:col10/1731883233429/Put/seqid=0 2024-11-17T22:40:34,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741998_1174 (size=12151) 2024-11-17T22:40:34,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T22:40:34,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:34,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:34,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:34,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T22:40:34,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:34,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:34,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:34,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883294617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:34,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883294617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:34,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883294627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883294627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:34,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T22:40:34,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:34,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:34,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:34,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:34,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/c235c3a89d4e43e599a4dcf4367252b7 2024-11-17T22:40:34,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/9ef13b608dc04a9783d2782a1ea3b1c4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9ef13b608dc04a9783d2782a1ea3b1c4 2024-11-17T22:40:34,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9ef13b608dc04a9783d2782a1ea3b1c4, entries=150, sequenceid=203, filesize=30.4 K 2024-11-17T22:40:34,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/db4bb70747cf4f22986205fbe5c90763 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/db4bb70747cf4f22986205fbe5c90763 2024-11-17T22:40:34,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/db4bb70747cf4f22986205fbe5c90763, entries=150, sequenceid=203, filesize=11.9 K 2024-11-17T22:40:34,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/c235c3a89d4e43e599a4dcf4367252b7 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c235c3a89d4e43e599a4dcf4367252b7 2024-11-17T22:40:34,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c235c3a89d4e43e599a4dcf4367252b7, entries=150, sequenceid=203, filesize=11.9 K 2024-11-17T22:40:34,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3893ec8e69968d25a1df1a69563cc10b in 1390ms, sequenceid=203, compaction requested=false 2024-11-17T22:40:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:34,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T22:40:34,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:34,908 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T22:40:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c772910f6896461d8998c9b34475e7e4_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883233476/Put/seqid=0 2024-11-17T22:40:34,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741999_1175 (size=12304) 2024-11-17T22:40:34,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:35,004 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c772910f6896461d8998c9b34475e7e4_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c772910f6896461d8998c9b34475e7e4_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/c4a929e139bc441ba3caa7b2dde2c00f, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:35,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/c4a929e139bc441ba3caa7b2dde2c00f is 175, key is test_row_0/A:col10/1731883233476/Put/seqid=0 2024-11-17T22:40:35,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742000_1176 (size=31105) 2024-11-17T22:40:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:35,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:35,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883295117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:35,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883295227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T22:40:35,432 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/c4a929e139bc441ba3caa7b2dde2c00f 2024-11-17T22:40:35,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883295437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/6d4fb7c0821a45669745d1a8aba56c86 is 50, key is test_row_0/B:col10/1731883233476/Put/seqid=0 2024-11-17T22:40:35,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742001_1177 (size=12151) 2024-11-17T22:40:35,466 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/6d4fb7c0821a45669745d1a8aba56c86 2024-11-17T22:40:35,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/de1af71ac2ae4a0a83da0701c3de8b28 is 50, key is test_row_0/C:col10/1731883233476/Put/seqid=0 2024-11-17T22:40:35,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742002_1178 (size=12151) 2024-11-17T22:40:35,489 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/de1af71ac2ae4a0a83da0701c3de8b28 2024-11-17T22:40:35,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/c4a929e139bc441ba3caa7b2dde2c00f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c4a929e139bc441ba3caa7b2dde2c00f 2024-11-17T22:40:35,511 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c4a929e139bc441ba3caa7b2dde2c00f, entries=150, sequenceid=227, filesize=30.4 K 2024-11-17T22:40:35,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/6d4fb7c0821a45669745d1a8aba56c86 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6d4fb7c0821a45669745d1a8aba56c86 2024-11-17T22:40:35,520 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6d4fb7c0821a45669745d1a8aba56c86, entries=150, sequenceid=227, filesize=11.9 K 2024-11-17T22:40:35,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/de1af71ac2ae4a0a83da0701c3de8b28 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/de1af71ac2ae4a0a83da0701c3de8b28 2024-11-17T22:40:35,530 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/de1af71ac2ae4a0a83da0701c3de8b28, entries=150, sequenceid=227, filesize=11.9 K 2024-11-17T22:40:35,532 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3893ec8e69968d25a1df1a69563cc10b in 624ms, sequenceid=227, compaction requested=true 2024-11-17T22:40:35,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:35,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:35,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-17T22:40:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-17T22:40:35,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-17T22:40:35,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3210 sec 2024-11-17T22:40:35,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.3300 sec 2024-11-17T22:40:35,629 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:35,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T22:40:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:35,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111711a15985e0ff44c6a0bba615ed17a6b8_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883235092/Put/seqid=0 2024-11-17T22:40:35,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742003_1179 (size=14794) 2024-11-17T22:40:35,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883295698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883295708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:35,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883295710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883295710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:35,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883295757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:35,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883295817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883295817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:35,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883295818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883295817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883296027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883296027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883296027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883296028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,048 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:36,066 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111711a15985e0ff44c6a0bba615ed17a6b8_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111711a15985e0ff44c6a0bba615ed17a6b8_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:36,067 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/48c22566eb31489f8c87d5b5c6ce86c3, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:36,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/48c22566eb31489f8c87d5b5c6ce86c3 is 175, key is test_row_0/A:col10/1731883235092/Put/seqid=0 2024-11-17T22:40:36,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742004_1180 (size=39749) 2024-11-17T22:40:36,116 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/48c22566eb31489f8c87d5b5c6ce86c3 2024-11-17T22:40:36,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/b901375f60f04e74889595fc7d5b6b14 is 50, key is test_row_0/B:col10/1731883235092/Put/seqid=0 2024-11-17T22:40:36,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742005_1181 (size=12151) 2024-11-17T22:40:36,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/b901375f60f04e74889595fc7d5b6b14 2024-11-17T22:40:36,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/9004f924f58e4450865e2352b7e68ef7 is 50, key is test_row_0/C:col10/1731883235092/Put/seqid=0 2024-11-17T22:40:36,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742006_1182 (size=12151) 2024-11-17T22:40:36,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883296277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883296337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883296337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883296337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883296337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/9004f924f58e4450865e2352b7e68ef7 2024-11-17T22:40:36,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/48c22566eb31489f8c87d5b5c6ce86c3 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/48c22566eb31489f8c87d5b5c6ce86c3 2024-11-17T22:40:36,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/48c22566eb31489f8c87d5b5c6ce86c3, entries=200, sequenceid=243, filesize=38.8 K 2024-11-17T22:40:36,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/b901375f60f04e74889595fc7d5b6b14 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/b901375f60f04e74889595fc7d5b6b14 2024-11-17T22:40:36,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/b901375f60f04e74889595fc7d5b6b14, entries=150, sequenceid=243, filesize=11.9 K 2024-11-17T22:40:36,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/9004f924f58e4450865e2352b7e68ef7 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/9004f924f58e4450865e2352b7e68ef7 2024-11-17T22:40:36,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/9004f924f58e4450865e2352b7e68ef7, entries=150, sequenceid=243, filesize=11.9 K 2024-11-17T22:40:36,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3893ec8e69968d25a1df1a69563cc10b in 1007ms, sequenceid=243, compaction requested=true 2024-11-17T22:40:36,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:36,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:36,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:36,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:36,636 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:36,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:36,636 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:36,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:36,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:36,637 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:36,637 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:36,637 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/B is initiating minor compaction (all files) 2024-11-17T22:40:36,637 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/A is initiating minor compaction (all files) 2024-11-17T22:40:36,638 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/A in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:36,638 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/B in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:36,638 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7e0dc59fa6e443a2bfdb67fb09e8a01e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9ef13b608dc04a9783d2782a1ea3b1c4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c4a929e139bc441ba3caa7b2dde2c00f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/48c22566eb31489f8c87d5b5c6ce86c3] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=130.3 K 2024-11-17T22:40:36,638 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/9d8d7f704a9d4e3d946d0773197bc7c5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/db4bb70747cf4f22986205fbe5c90763, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6d4fb7c0821a45669745d1a8aba56c86, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/b901375f60f04e74889595fc7d5b6b14] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=47.8 K 2024-11-17T22:40:36,638 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:36,638 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7e0dc59fa6e443a2bfdb67fb09e8a01e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9ef13b608dc04a9783d2782a1ea3b1c4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c4a929e139bc441ba3caa7b2dde2c00f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/48c22566eb31489f8c87d5b5c6ce86c3] 2024-11-17T22:40:36,638 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d8d7f704a9d4e3d946d0773197bc7c5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731883232783 2024-11-17T22:40:36,638 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e0dc59fa6e443a2bfdb67fb09e8a01e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731883232783 2024-11-17T22:40:36,638 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting db4bb70747cf4f22986205fbe5c90763, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731883233428 2024-11-17T22:40:36,638 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ef13b608dc04a9783d2782a1ea3b1c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731883233428 2024-11-17T22:40:36,639 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d4fb7c0821a45669745d1a8aba56c86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731883233451 2024-11-17T22:40:36,639 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4a929e139bc441ba3caa7b2dde2c00f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731883233451 2024-11-17T22:40:36,639 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c22566eb31489f8c87d5b5c6ce86c3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731883235092 2024-11-17T22:40:36,639 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b901375f60f04e74889595fc7d5b6b14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731883235092 2024-11-17T22:40:36,651 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#B#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:36,651 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/f6b5c2b496f54dfcb48b3d421961659f is 50, key is test_row_0/B:col10/1731883235092/Put/seqid=0 2024-11-17T22:40:36,655 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:36,666 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117f8194394ad73446aa39fe351ae3bf6ee_3893ec8e69968d25a1df1a69563cc10b store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:36,669 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117f8194394ad73446aa39fe351ae3bf6ee_3893ec8e69968d25a1df1a69563cc10b, store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:36,669 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f8194394ad73446aa39fe351ae3bf6ee_3893ec8e69968d25a1df1a69563cc10b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:36,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742007_1183 (size=12663) 2024-11-17T22:40:36,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742008_1184 (size=4469) 2024-11-17T22:40:36,677 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/f6b5c2b496f54dfcb48b3d421961659f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f6b5c2b496f54dfcb48b3d421961659f 2024-11-17T22:40:36,678 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#A#compaction#153 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:36,679 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/c5ba1145812f4868ab65fbf8f50aa83f is 175, key is test_row_0/A:col10/1731883235092/Put/seqid=0 2024-11-17T22:40:36,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742009_1185 (size=31617) 2024-11-17T22:40:36,686 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/B of 3893ec8e69968d25a1df1a69563cc10b into f6b5c2b496f54dfcb48b3d421961659f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:36,686 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:36,686 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/B, priority=12, startTime=1731883236636; duration=0sec 2024-11-17T22:40:36,686 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:36,686 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:B 2024-11-17T22:40:36,686 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:36,689 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:36,689 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/C is initiating minor compaction (all files) 2024-11-17T22:40:36,689 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/C in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:36,689 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2fa920313afc4096b289bd219d3664b3, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c235c3a89d4e43e599a4dcf4367252b7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/de1af71ac2ae4a0a83da0701c3de8b28, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/9004f924f58e4450865e2352b7e68ef7] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=47.8 K 2024-11-17T22:40:36,690 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fa920313afc4096b289bd219d3664b3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731883232783 2024-11-17T22:40:36,690 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/c5ba1145812f4868ab65fbf8f50aa83f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c5ba1145812f4868ab65fbf8f50aa83f 2024-11-17T22:40:36,691 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting c235c3a89d4e43e599a4dcf4367252b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731883233428 2024-11-17T22:40:36,691 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting de1af71ac2ae4a0a83da0701c3de8b28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731883233451 2024-11-17T22:40:36,691 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 9004f924f58e4450865e2352b7e68ef7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731883235092 2024-11-17T22:40:36,698 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/A of 3893ec8e69968d25a1df1a69563cc10b into c5ba1145812f4868ab65fbf8f50aa83f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:36,698 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:36,698 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/A, priority=12, startTime=1731883236636; duration=0sec 2024-11-17T22:40:36,698 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:36,698 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:A 2024-11-17T22:40:36,704 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#C#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:36,704 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/4379f5c825bd478298533d5207bab9f6 is 50, key is test_row_0/C:col10/1731883235092/Put/seqid=0 2024-11-17T22:40:36,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742010_1186 (size=12663) 2024-11-17T22:40:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:36,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T22:40:36,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111742dfe5271a2e4292bba522bf761b30b8_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883236847/Put/seqid=0 2024-11-17T22:40:36,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883296870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883296874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883296876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883296876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742011_1187 (size=14994) 2024-11-17T22:40:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883296987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883296987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883296991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:36,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883296997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:37,114 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/4379f5c825bd478298533d5207bab9f6 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/4379f5c825bd478298533d5207bab9f6 2024-11-17T22:40:37,119 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/C of 3893ec8e69968d25a1df1a69563cc10b into 4379f5c825bd478298533d5207bab9f6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:37,119 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:37,119 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/C, priority=12, startTime=1731883236636; duration=0sec 2024-11-17T22:40:37,119 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:37,119 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:C 2024-11-17T22:40:37,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883297196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:37,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883297196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:37,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883297196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:37,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883297207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:37,283 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:37,287 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111742dfe5271a2e4292bba522bf761b30b8_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111742dfe5271a2e4292bba522bf761b30b8_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:37,288 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/0bb8b1a84ea2421c8472deccdebc776e, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:37,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/0bb8b1a84ea2421c8472deccdebc776e is 175, key is test_row_0/A:col10/1731883236847/Put/seqid=0 2024-11-17T22:40:37,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742012_1188 (size=39949) 2024-11-17T22:40:37,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883297296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:37,301 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in 3893ec8e69968d25a1df1a69563cc10b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:40:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T22:40:37,377 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:37,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-17T22:40:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T22:40:37,382 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:37,383 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:37,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T22:40:37,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:37,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883297506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883297507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:37,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883297507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:37,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883297517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:37,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T22:40:37,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:37,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:37,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:37,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:37,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:37,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T22:40:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:37,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:37,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/0bb8b1a84ea2421c8472deccdebc776e 2024-11-17T22:40:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T22:40:37,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/f0257fbb86a04dc9b2797467c02c3c1f is 50, key is test_row_0/B:col10/1731883236847/Put/seqid=0 2024-11-17T22:40:37,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742013_1189 (size=12301) 2024-11-17T22:40:37,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/f0257fbb86a04dc9b2797467c02c3c1f 2024-11-17T22:40:37,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/2e0f35306c1e473fa20d0597a7a4529a is 50, key is test_row_0/C:col10/1731883236847/Put/seqid=0 2024-11-17T22:40:37,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742014_1190 (size=12301) 2024-11-17T22:40:37,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/2e0f35306c1e473fa20d0597a7a4529a 2024-11-17T22:40:37,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/0bb8b1a84ea2421c8472deccdebc776e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/0bb8b1a84ea2421c8472deccdebc776e 2024-11-17T22:40:37,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/0bb8b1a84ea2421c8472deccdebc776e, entries=200, sequenceid=266, filesize=39.0 K 2024-11-17T22:40:37,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/f0257fbb86a04dc9b2797467c02c3c1f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f0257fbb86a04dc9b2797467c02c3c1f 2024-11-17T22:40:37,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f0257fbb86a04dc9b2797467c02c3c1f, entries=150, sequenceid=266, filesize=12.0 K 2024-11-17T22:40:37,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/2e0f35306c1e473fa20d0597a7a4529a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2e0f35306c1e473fa20d0597a7a4529a 2024-11-17T22:40:37,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2e0f35306c1e473fa20d0597a7a4529a, entries=150, sequenceid=266, filesize=12.0 K 2024-11-17T22:40:37,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3893ec8e69968d25a1df1a69563cc10b in 909ms, sequenceid=266, compaction requested=false 2024-11-17T22:40:37,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:37,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T22:40:37,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:37,842 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T22:40:37,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b8b58d1fa67d42968b7ededd3a9d5b4e_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883236860/Put/seqid=0 2024-11-17T22:40:37,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742015_1191 (size=12454) 2024-11-17T22:40:37,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:37,900 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b8b58d1fa67d42968b7ededd3a9d5b4e_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b8b58d1fa67d42968b7ededd3a9d5b4e_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:37,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/3a436e768dda42b6b1126313de4bb374, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:37,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/3a436e768dda42b6b1126313de4bb374 is 175, key is test_row_0/A:col10/1731883236860/Put/seqid=0 2024-11-17T22:40:37,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742016_1192 (size=31255) 2024-11-17T22:40:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T22:40:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:38,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:38,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883298035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883298038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883298040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883298041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883298147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883298147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883298147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883298157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,316 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/3a436e768dda42b6b1126313de4bb374 2024-11-17T22:40:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/c8c92e340b874e47a06641267c7b1c1f is 50, key is test_row_0/B:col10/1731883236860/Put/seqid=0 2024-11-17T22:40:38,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742017_1193 (size=12301) 2024-11-17T22:40:38,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883298357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883298357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883298357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883298367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T22:40:38,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883298667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883298667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883298667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883298677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:38,736 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/c8c92e340b874e47a06641267c7b1c1f 2024-11-17T22:40:38,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/c65dd6c563444be2bb6ecea3f3f5f32f is 50, key is test_row_0/C:col10/1731883236860/Put/seqid=0 2024-11-17T22:40:38,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742018_1194 (size=12301) 2024-11-17T22:40:39,146 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/c65dd6c563444be2bb6ecea3f3f5f32f 2024-11-17T22:40:39,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/3a436e768dda42b6b1126313de4bb374 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/3a436e768dda42b6b1126313de4bb374 2024-11-17T22:40:39,155 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/3a436e768dda42b6b1126313de4bb374, entries=150, sequenceid=282, filesize=30.5 K 2024-11-17T22:40:39,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/c8c92e340b874e47a06641267c7b1c1f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c8c92e340b874e47a06641267c7b1c1f 2024-11-17T22:40:39,161 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c8c92e340b874e47a06641267c7b1c1f, entries=150, sequenceid=282, filesize=12.0 K 2024-11-17T22:40:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/c65dd6c563444be2bb6ecea3f3f5f32f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c65dd6c563444be2bb6ecea3f3f5f32f 2024-11-17T22:40:39,166 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c65dd6c563444be2bb6ecea3f3f5f32f, entries=150, sequenceid=282, filesize=12.0 K 2024-11-17T22:40:39,167 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3893ec8e69968d25a1df1a69563cc10b in 1325ms, sequenceid=282, compaction requested=true 2024-11-17T22:40:39,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:39,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:39,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-17T22:40:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-17T22:40:39,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-17T22:40:39,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7850 sec 2024-11-17T22:40:39,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.7930 sec 2024-11-17T22:40:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:39,178 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:39,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T22:40:39,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174fd71970a55c4a8bab30af6fa66ee49b_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883239177/Put/seqid=0 2024-11-17T22:40:39,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742019_1195 (size=14994) 2024-11-17T22:40:39,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883299202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883299202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883299203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883299203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51520 deadline: 1731883299307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,310 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in 3893ec8e69968d25a1df1a69563cc10b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:40:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883299316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883299316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883299316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883299316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T22:40:39,527 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:39,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883299526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883299527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883299527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883299527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-17T22:40:39,529 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:39,529 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T22:40:39,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:39,594 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:39,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174fd71970a55c4a8bab30af6fa66ee49b_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174fd71970a55c4a8bab30af6fa66ee49b_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:39,600 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/69635a37111c46938ebdc115acae9a00, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:39,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/69635a37111c46938ebdc115acae9a00 is 175, key is test_row_0/A:col10/1731883239177/Put/seqid=0 2024-11-17T22:40:39,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742020_1196 (size=39949) 2024-11-17T22:40:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T22:40:39,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T22:40:39,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:39,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:39,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:39,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:39,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:39,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T22:40:39,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:39,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:39,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:39,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:39,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:39,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883299837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883299837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883299837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883299837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T22:40:39,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T22:40:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:39,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,007 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/69635a37111c46938ebdc115acae9a00 2024-11-17T22:40:40,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/6b343b7b543e4f059eec7fc2f028ef29 is 50, key is test_row_0/B:col10/1731883239177/Put/seqid=0 2024-11-17T22:40:40,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742021_1197 (size=12301) 2024-11-17T22:40:40,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T22:40:40,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:40,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T22:40:40,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T22:40:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883300347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:40,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883300347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:40,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883300347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:40,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883300347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:40,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/6b343b7b543e4f059eec7fc2f028ef29 2024-11-17T22:40:40,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/a3bde199b3f44937b17d891906e30a5a is 50, key is test_row_0/C:col10/1731883239177/Put/seqid=0 2024-11-17T22:40:40,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742022_1198 (size=12301) 2024-11-17T22:40:40,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T22:40:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T22:40:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T22:40:40,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T22:40:40,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:40,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:40,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/a3bde199b3f44937b17d891906e30a5a 2024-11-17T22:40:40,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/69635a37111c46938ebdc115acae9a00 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/69635a37111c46938ebdc115acae9a00 2024-11-17T22:40:40,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/69635a37111c46938ebdc115acae9a00, entries=200, sequenceid=304, filesize=39.0 K 2024-11-17T22:40:40,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/6b343b7b543e4f059eec7fc2f028ef29 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6b343b7b543e4f059eec7fc2f028ef29 2024-11-17T22:40:40,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6b343b7b543e4f059eec7fc2f028ef29, entries=150, sequenceid=304, filesize=12.0 K 2024-11-17T22:40:40,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/a3bde199b3f44937b17d891906e30a5a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a3bde199b3f44937b17d891906e30a5a 2024-11-17T22:40:40,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a3bde199b3f44937b17d891906e30a5a, entries=150, sequenceid=304, filesize=12.0 K 2024-11-17T22:40:40,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 3893ec8e69968d25a1df1a69563cc10b in 1694ms, sequenceid=304, compaction requested=true 2024-11-17T22:40:40,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:40,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:40,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:40,873 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:40,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:40,873 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:40,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:40,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:40,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:40,874 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:40,874 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:40,874 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/B is initiating minor compaction (all files) 2024-11-17T22:40:40,874 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/A is initiating minor compaction (all files) 2024-11-17T22:40:40,874 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/B in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,874 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/A in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,874 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c5ba1145812f4868ab65fbf8f50aa83f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/0bb8b1a84ea2421c8472deccdebc776e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/3a436e768dda42b6b1126313de4bb374, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/69635a37111c46938ebdc115acae9a00] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=139.4 K 2024-11-17T22:40:40,874 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f6b5c2b496f54dfcb48b3d421961659f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f0257fbb86a04dc9b2797467c02c3c1f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c8c92e340b874e47a06641267c7b1c1f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6b343b7b543e4f059eec7fc2f028ef29] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=48.4 K 2024-11-17T22:40:40,874 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,874 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c5ba1145812f4868ab65fbf8f50aa83f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/0bb8b1a84ea2421c8472deccdebc776e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/3a436e768dda42b6b1126313de4bb374, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/69635a37111c46938ebdc115acae9a00] 2024-11-17T22:40:40,875 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting f6b5c2b496f54dfcb48b3d421961659f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731883235092 2024-11-17T22:40:40,875 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5ba1145812f4868ab65fbf8f50aa83f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731883235092 2024-11-17T22:40:40,875 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting f0257fbb86a04dc9b2797467c02c3c1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731883235707 2024-11-17T22:40:40,875 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bb8b1a84ea2421c8472deccdebc776e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731883235696 2024-11-17T22:40:40,875 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting c8c92e340b874e47a06641267c7b1c1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731883236860 2024-11-17T22:40:40,875 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a436e768dda42b6b1126313de4bb374, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731883236860 2024-11-17T22:40:40,875 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69635a37111c46938ebdc115acae9a00, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731883238031 2024-11-17T22:40:40,875 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b343b7b543e4f059eec7fc2f028ef29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731883238036 2024-11-17T22:40:40,882 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:40,885 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411174a82d28ecdad43fda31a62cc0e5aec4f_3893ec8e69968d25a1df1a69563cc10b store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:40,886 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:40,886 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/5ff9952def0345d3b4ed2bbe82c80a5a is 50, key is test_row_0/B:col10/1731883239177/Put/seqid=0 2024-11-17T22:40:40,888 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411174a82d28ecdad43fda31a62cc0e5aec4f_3893ec8e69968d25a1df1a69563cc10b, store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:40,888 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174a82d28ecdad43fda31a62cc0e5aec4f_3893ec8e69968d25a1df1a69563cc10b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:40,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T22:40:40,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:40,910 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T22:40:40,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742023_1199 (size=12949) 2024-11-17T22:40:40,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742024_1200 (size=4469) 2024-11-17T22:40:40,919 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#A#compaction#164 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:40,919 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/187bcc1775d2451a9ba09922eacf4c7f is 175, key is test_row_0/A:col10/1731883239177/Put/seqid=0 2024-11-17T22:40:40,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742025_1201 (size=31903) 2024-11-17T22:40:40,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111739f7a2a0ccdf4400ac65b7161b6247df_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883239201/Put/seqid=0 2024-11-17T22:40:40,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742026_1202 (size=12454) 2024-11-17T22:40:40,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:40,938 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111739f7a2a0ccdf4400ac65b7161b6247df_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111739f7a2a0ccdf4400ac65b7161b6247df_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:40,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/7c3a67196b0b4bcd84bbc7c1aafaf64a, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:40,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/7c3a67196b0b4bcd84bbc7c1aafaf64a is 175, key is test_row_0/A:col10/1731883239201/Put/seqid=0 2024-11-17T22:40:40,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742027_1203 (size=31255) 2024-11-17T22:40:40,944 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/7c3a67196b0b4bcd84bbc7c1aafaf64a 2024-11-17T22:40:40,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/4625364b15fb49aca484485b955bb9c4 is 50, key is test_row_0/B:col10/1731883239201/Put/seqid=0 2024-11-17T22:40:40,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742028_1204 (size=12301) 2024-11-17T22:40:41,320 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/5ff9952def0345d3b4ed2bbe82c80a5a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/5ff9952def0345d3b4ed2bbe82c80a5a 2024-11-17T22:40:41,325 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/B of 3893ec8e69968d25a1df1a69563cc10b into 5ff9952def0345d3b4ed2bbe82c80a5a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:41,325 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:41,325 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/B, priority=12, startTime=1731883240873; duration=0sec 2024-11-17T22:40:41,325 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:41,326 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:B 2024-11-17T22:40:41,326 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:41,327 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:41,327 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/C is initiating minor compaction (all files) 2024-11-17T22:40:41,327 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/C in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:41,327 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/4379f5c825bd478298533d5207bab9f6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2e0f35306c1e473fa20d0597a7a4529a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c65dd6c563444be2bb6ecea3f3f5f32f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a3bde199b3f44937b17d891906e30a5a] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=48.4 K 2024-11-17T22:40:41,327 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4379f5c825bd478298533d5207bab9f6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731883235092 2024-11-17T22:40:41,328 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e0f35306c1e473fa20d0597a7a4529a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731883235707 2024-11-17T22:40:41,328 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting c65dd6c563444be2bb6ecea3f3f5f32f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731883236860 2024-11-17T22:40:41,328 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting a3bde199b3f44937b17d891906e30a5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731883238036 2024-11-17T22:40:41,334 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/187bcc1775d2451a9ba09922eacf4c7f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/187bcc1775d2451a9ba09922eacf4c7f 2024-11-17T22:40:41,340 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:41,340 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/A of 3893ec8e69968d25a1df1a69563cc10b into 187bcc1775d2451a9ba09922eacf4c7f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:41,340 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:41,341 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/A, priority=12, startTime=1731883240873; duration=0sec 2024-11-17T22:40:41,341 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:41,341 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:A 2024-11-17T22:40:41,341 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/fda1a7fd42fe4d5996feaa19784fa103 is 50, key is test_row_0/C:col10/1731883239177/Put/seqid=0 2024-11-17T22:40:41,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742029_1205 (size=12949) 2024-11-17T22:40:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:41,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:41,367 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/4625364b15fb49aca484485b955bb9c4 2024-11-17T22:40:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/b811e8190b774e5c90e8450a98326950 is 50, key is test_row_0/C:col10/1731883239201/Put/seqid=0 2024-11-17T22:40:41,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883301403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883301404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883301405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742030_1206 (size=12301) 2024-11-17T22:40:41,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883301406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,412 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/b811e8190b774e5c90e8450a98326950 2024-11-17T22:40:41,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/7c3a67196b0b4bcd84bbc7c1aafaf64a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7c3a67196b0b4bcd84bbc7c1aafaf64a 2024-11-17T22:40:41,428 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7c3a67196b0b4bcd84bbc7c1aafaf64a, entries=150, sequenceid=319, filesize=30.5 K 2024-11-17T22:40:41,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/4625364b15fb49aca484485b955bb9c4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/4625364b15fb49aca484485b955bb9c4 2024-11-17T22:40:41,435 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/4625364b15fb49aca484485b955bb9c4, entries=150, sequenceid=319, filesize=12.0 K 2024-11-17T22:40:41,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/b811e8190b774e5c90e8450a98326950 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/b811e8190b774e5c90e8450a98326950 2024-11-17T22:40:41,444 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/b811e8190b774e5c90e8450a98326950, entries=150, sequenceid=319, filesize=12.0 K 2024-11-17T22:40:41,445 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3893ec8e69968d25a1df1a69563cc10b in 535ms, sequenceid=319, compaction requested=false 2024-11-17T22:40:41,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:41,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:41,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-17T22:40:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-17T22:40:41,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-17T22:40:41,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9180 sec 2024-11-17T22:40:41,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.9220 sec 2024-11-17T22:40:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:41,518 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:41,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T22:40:41,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173c32589684ef4b6b80bc35475bffaeb2_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883241518/Put/seqid=0 2024-11-17T22:40:41,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742031_1207 (size=17534) 2024-11-17T22:40:41,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,531 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883301529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883301530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883301530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,536 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173c32589684ef4b6b80bc35475bffaeb2_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173c32589684ef4b6b80bc35475bffaeb2_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:41,536 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/4dfb5e407e7a489fba491ca39bfa48e0, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:41,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883301532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/4dfb5e407e7a489fba491ca39bfa48e0 is 175, key is test_row_0/A:col10/1731883241518/Put/seqid=0 2024-11-17T22:40:41,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742032_1208 (size=48639) 2024-11-17T22:40:41,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883301637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883301637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883301637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883301647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T22:40:41,677 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:41,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-17T22:40:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T22:40:41,679 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:41,679 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:41,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:41,757 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/fda1a7fd42fe4d5996feaa19784fa103 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/fda1a7fd42fe4d5996feaa19784fa103 2024-11-17T22:40:41,763 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/C of 3893ec8e69968d25a1df1a69563cc10b into fda1a7fd42fe4d5996feaa19784fa103(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:41,763 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:41,763 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/C, priority=12, startTime=1731883240873; duration=0sec 2024-11-17T22:40:41,763 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:41,763 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:C 2024-11-17T22:40:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T22:40:41,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T22:40:41,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:41,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:41,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:41,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:41,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:41,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883301847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883301847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883301847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883301857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:41,946 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/4dfb5e407e7a489fba491ca39bfa48e0 2024-11-17T22:40:41,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/37cf353006704656bbb7ea63d60d66f0 is 50, key is test_row_0/B:col10/1731883241518/Put/seqid=0 2024-11-17T22:40:41,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742033_1209 (size=12301) 2024-11-17T22:40:41,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T22:40:41,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:41,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:41,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:41,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:41,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T22:40:42,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T22:40:42,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:42,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883302157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883302157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883302157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:42,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883302166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:42,200 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-17T22:40:42,200 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:42,200 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:42,201 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:42,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:42,203 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-17T22:40:42,203 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:42,203 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:42,203 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:42,203 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:42,205 INFO [Thread-631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-631. 2024-11-17T22:40:42,205 DEBUG [Thread-631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:42,205 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:42,205 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:42,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:42,207 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-17T22:40:42,207 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:42,207 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:42,207 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:42,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:42,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T22:40:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T22:40:42,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/37cf353006704656bbb7ea63d60d66f0 2024-11-17T22:40:42,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/bfe3f54374cf4a4785fd5e358d852056 is 50, key is test_row_0/C:col10/1731883241518/Put/seqid=0 2024-11-17T22:40:42,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742034_1210 (size=12301) 2024-11-17T22:40:42,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T22:40:42,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:42,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T22:40:42,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:42,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:42,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731883302666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:42,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51538 deadline: 1731883302667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51500 deadline: 1731883302667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:42,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51524 deadline: 1731883302677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:40:42,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T22:40:42,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:42,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:42,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/bfe3f54374cf4a4785fd5e358d852056 2024-11-17T22:40:42,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/4dfb5e407e7a489fba491ca39bfa48e0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/4dfb5e407e7a489fba491ca39bfa48e0 2024-11-17T22:40:42,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/4dfb5e407e7a489fba491ca39bfa48e0, entries=250, sequenceid=344, filesize=47.5 K 2024-11-17T22:40:42,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/37cf353006704656bbb7ea63d60d66f0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/37cf353006704656bbb7ea63d60d66f0 2024-11-17T22:40:42,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/37cf353006704656bbb7ea63d60d66f0, entries=150, sequenceid=344, filesize=12.0 K 2024-11-17T22:40:42,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/bfe3f54374cf4a4785fd5e358d852056 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/bfe3f54374cf4a4785fd5e358d852056 2024-11-17T22:40:42,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/bfe3f54374cf4a4785fd5e358d852056, entries=150, sequenceid=344, filesize=12.0 K 2024-11-17T22:40:42,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 3893ec8e69968d25a1df1a69563cc10b in 1291ms, sequenceid=344, compaction requested=true 2024-11-17T22:40:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:42,809 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:42,809 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3893ec8e69968d25a1df1a69563cc10b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:42,810 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:42,810 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:42,811 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/B is initiating minor compaction (all files) 2024-11-17T22:40:42,811 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/A is initiating minor compaction (all files) 2024-11-17T22:40:42,811 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/B in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,811 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/A in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,811 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/5ff9952def0345d3b4ed2bbe82c80a5a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/4625364b15fb49aca484485b955bb9c4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/37cf353006704656bbb7ea63d60d66f0] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=36.7 K 2024-11-17T22:40:42,811 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/187bcc1775d2451a9ba09922eacf4c7f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7c3a67196b0b4bcd84bbc7c1aafaf64a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/4dfb5e407e7a489fba491ca39bfa48e0] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=109.2 K 2024-11-17T22:40:42,811 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,811 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/187bcc1775d2451a9ba09922eacf4c7f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7c3a67196b0b4bcd84bbc7c1aafaf64a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/4dfb5e407e7a489fba491ca39bfa48e0] 2024-11-17T22:40:42,811 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ff9952def0345d3b4ed2bbe82c80a5a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731883238036 2024-11-17T22:40:42,811 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 187bcc1775d2451a9ba09922eacf4c7f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731883238036 2024-11-17T22:40:42,812 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4625364b15fb49aca484485b955bb9c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731883239200 2024-11-17T22:40:42,812 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c3a67196b0b4bcd84bbc7c1aafaf64a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731883239200 2024-11-17T22:40:42,812 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 37cf353006704656bbb7ea63d60d66f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731883241397 2024-11-17T22:40:42,812 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dfb5e407e7a489fba491ca39bfa48e0, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731883241397 2024-11-17T22:40:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T22:40:42,823 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#B#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:42,823 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/c6270c4177ae428cba75933d9537d70f is 50, key is test_row_0/B:col10/1731883241518/Put/seqid=0 2024-11-17T22:40:42,826 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:42,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742035_1211 (size=13051) 2024-11-17T22:40:42,828 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111712fbb28da2e14911b6e864d42a499c80_3893ec8e69968d25a1df1a69563cc10b store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:42,854 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111712fbb28da2e14911b6e864d42a499c80_3893ec8e69968d25a1df1a69563cc10b, store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:42,854 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111712fbb28da2e14911b6e864d42a499c80_3893ec8e69968d25a1df1a69563cc10b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:42,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742036_1212 (size=4469) 2024-11-17T22:40:42,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T22:40:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:42,910 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:40:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111769d6bf2085fb4ed49ec539f3111d266d_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883241529/Put/seqid=0 2024-11-17T22:40:42,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742037_1213 (size=12454) 2024-11-17T22:40:42,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:42,930 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111769d6bf2085fb4ed49ec539f3111d266d_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111769d6bf2085fb4ed49ec539f3111d266d_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:42,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/33a1d9e2a0374b8086a6f12f8af04723, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:42,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/33a1d9e2a0374b8086a6f12f8af04723 is 175, key is test_row_0/A:col10/1731883241529/Put/seqid=0 2024-11-17T22:40:42,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742038_1214 (size=31255) 2024-11-17T22:40:43,236 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/c6270c4177ae428cba75933d9537d70f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c6270c4177ae428cba75933d9537d70f 2024-11-17T22:40:43,242 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/B of 3893ec8e69968d25a1df1a69563cc10b into c6270c4177ae428cba75933d9537d70f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:43,242 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:43,242 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/B, priority=13, startTime=1731883242809; duration=0sec 2024-11-17T22:40:43,242 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:43,243 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:B 2024-11-17T22:40:43,243 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:43,244 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:43,244 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 3893ec8e69968d25a1df1a69563cc10b/C is initiating minor compaction (all files) 2024-11-17T22:40:43,244 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3893ec8e69968d25a1df1a69563cc10b/C in TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:43,244 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/fda1a7fd42fe4d5996feaa19784fa103, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/b811e8190b774e5c90e8450a98326950, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/bfe3f54374cf4a4785fd5e358d852056] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp, totalSize=36.7 K 2024-11-17T22:40:43,244 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting fda1a7fd42fe4d5996feaa19784fa103, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731883238036 2024-11-17T22:40:43,244 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b811e8190b774e5c90e8450a98326950, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731883239200 2024-11-17T22:40:43,245 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe3f54374cf4a4785fd5e358d852056, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731883241397 2024-11-17T22:40:43,251 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#C#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:43,251 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/ab8a4642b3cb40c2810b256160b0a512 is 50, key is test_row_0/C:col10/1731883241518/Put/seqid=0 2024-11-17T22:40:43,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742039_1215 (size=13051) 2024-11-17T22:40:43,263 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3893ec8e69968d25a1df1a69563cc10b#A#compaction#174 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:43,263 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/78268150829e4af28c86dc649a9c8506 is 175, key is test_row_0/A:col10/1731883241518/Put/seqid=0 2024-11-17T22:40:43,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742040_1216 (size=32005) 2024-11-17T22:40:43,336 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/33a1d9e2a0374b8086a6f12f8af04723 2024-11-17T22:40:43,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/a8cd45847b144aa29f24356465e30812 is 50, key is test_row_0/B:col10/1731883241529/Put/seqid=0 2024-11-17T22:40:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. as already flushing 2024-11-17T22:40:43,349 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-17T22:40:43,349 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:43,349 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:43,349 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:43,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:43,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742041_1217 (size=12301) 2024-11-17T22:40:43,665 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/ab8a4642b3cb40c2810b256160b0a512 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ab8a4642b3cb40c2810b256160b0a512 2024-11-17T22:40:43,671 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/C of 3893ec8e69968d25a1df1a69563cc10b into ab8a4642b3cb40c2810b256160b0a512(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:43,671 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:43,671 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/C, priority=13, startTime=1731883242809; duration=0sec 2024-11-17T22:40:43,671 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:43,671 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:C 2024-11-17T22:40:43,672 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/78268150829e4af28c86dc649a9c8506 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/78268150829e4af28c86dc649a9c8506 2024-11-17T22:40:43,677 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3893ec8e69968d25a1df1a69563cc10b/A of 3893ec8e69968d25a1df1a69563cc10b into 78268150829e4af28c86dc649a9c8506(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:43,677 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:43,678 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., storeName=3893ec8e69968d25a1df1a69563cc10b/A, priority=13, startTime=1731883242809; duration=0sec 2024-11-17T22:40:43,678 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:43,678 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3893ec8e69968d25a1df1a69563cc10b:A 2024-11-17T22:40:43,678 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-17T22:40:43,678 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:43,678 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:43,678 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-11-17T22:40:43,678 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:43,678 INFO [Thread-618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-618. 2024-11-17T22:40:43,678 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:43,678 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:43,678 DEBUG [Thread-618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:43,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:43,678 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:43,678 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:43,678 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:43,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:43,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:43,689 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-17T22:40:43,689 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:40:43,689 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:43,689 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:43,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:43,752 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/a8cd45847b144aa29f24356465e30812 2024-11-17T22:40:43,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/3d70966c94ce4e4f8292c7b3db79dfde is 50, key is test_row_0/C:col10/1731883241529/Put/seqid=0 2024-11-17T22:40:43,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742042_1218 (size=12301) 2024-11-17T22:40:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T22:40:44,167 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/3d70966c94ce4e4f8292c7b3db79dfde 2024-11-17T22:40:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/33a1d9e2a0374b8086a6f12f8af04723 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/33a1d9e2a0374b8086a6f12f8af04723 2024-11-17T22:40:44,182 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/33a1d9e2a0374b8086a6f12f8af04723, entries=150, sequenceid=358, filesize=30.5 K 2024-11-17T22:40:44,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/a8cd45847b144aa29f24356465e30812 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/a8cd45847b144aa29f24356465e30812 2024-11-17T22:40:44,187 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/a8cd45847b144aa29f24356465e30812, entries=150, sequenceid=358, filesize=12.0 K 2024-11-17T22:40:44,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/3d70966c94ce4e4f8292c7b3db79dfde as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3d70966c94ce4e4f8292c7b3db79dfde 2024-11-17T22:40:44,192 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3d70966c94ce4e4f8292c7b3db79dfde, entries=150, sequenceid=358, filesize=12.0 K 2024-11-17T22:40:44,193 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=33.54 KB/34350 for 3893ec8e69968d25a1df1a69563cc10b in 1283ms, sequenceid=358, compaction requested=false 2024-11-17T22:40:44,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 3893ec8e69968d25a1df1a69563cc10b: 2024-11-17T22:40:44,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:44,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-17T22:40:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-17T22:40:44,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-17T22:40:44,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5160 sec 2024-11-17T22:40:44,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.5200 sec 2024-11-17T22:40:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T22:40:45,858 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2934 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2932 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1304 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3910 rows 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1312 2024-11-17T22:40:45,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3936 rows 2024-11-17T22:40:45,858 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T22:40:45,858 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T22:40:45,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:45,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:45,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:40:45,876 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T22:40:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T22:40:45,879 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883245879"}]},"ts":"1731883245879"} 2024-11-17T22:40:45,880 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T22:40:45,880 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T22:40:45,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T22:40:45,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, UNASSIGN}] 2024-11-17T22:40:45,882 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, UNASSIGN 2024-11-17T22:40:45,883 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=3893ec8e69968d25a1df1a69563cc10b, regionState=CLOSING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:45,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, UNASSIGN because future has completed 2024-11-17T22:40:45,884 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T22:40:45,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:40:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T22:40:46,037 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] handler.UnassignRegionHandler(122): Close 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:46,037 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T22:40:46,037 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1722): Closing 3893ec8e69968d25a1df1a69563cc10b, disabling compactions & flushes 2024-11-17T22:40:46,037 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:46,038 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:46,038 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. after waiting 0 ms 2024-11-17T22:40:46,038 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:46,038 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(2902): Flushing 3893ec8e69968d25a1df1a69563cc10b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T22:40:46,050 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176fd971e6a8ec4bfd8d2747ecf7407d2d_3893ec8e69968d25a1df1a69563cc10b is 50, key is test_row_0/A:col10/1731883243688/Put/seqid=0 2024-11-17T22:40:46,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742043_1219 (size=12454) 2024-11-17T22:40:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T22:40:46,455 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:46,462 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176fd971e6a8ec4bfd8d2747ecf7407d2d_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176fd971e6a8ec4bfd8d2747ecf7407d2d_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:46,464 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/ff041b8cabd8484bb4d927e1ac2c46f2, store: [table=TestAcidGuarantees family=A region=3893ec8e69968d25a1df1a69563cc10b] 2024-11-17T22:40:46,465 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/ff041b8cabd8484bb4d927e1ac2c46f2 is 175, key is test_row_0/A:col10/1731883243688/Put/seqid=0 2024-11-17T22:40:46,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742044_1220 (size=31255) 2024-11-17T22:40:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T22:40:46,869 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/ff041b8cabd8484bb4d927e1ac2c46f2 2024-11-17T22:40:46,882 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/f965de5326e54359911a6ac85929b5d5 is 50, key is test_row_0/B:col10/1731883243688/Put/seqid=0 2024-11-17T22:40:46,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742045_1221 (size=12301) 2024-11-17T22:40:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T22:40:47,246 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T22:40:47,288 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/f965de5326e54359911a6ac85929b5d5 2024-11-17T22:40:47,295 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/eeea81bec4b74ba88da9f50897fefadd is 50, key is test_row_0/C:col10/1731883243688/Put/seqid=0 2024-11-17T22:40:47,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742046_1222 (size=12301) 2024-11-17T22:40:47,705 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/eeea81bec4b74ba88da9f50897fefadd 2024-11-17T22:40:47,714 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/A/ff041b8cabd8484bb4d927e1ac2c46f2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/ff041b8cabd8484bb4d927e1ac2c46f2 2024-11-17T22:40:47,718 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/ff041b8cabd8484bb4d927e1ac2c46f2, entries=150, sequenceid=369, filesize=30.5 K 2024-11-17T22:40:47,719 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/B/f965de5326e54359911a6ac85929b5d5 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f965de5326e54359911a6ac85929b5d5 2024-11-17T22:40:47,724 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f965de5326e54359911a6ac85929b5d5, entries=150, sequenceid=369, filesize=12.0 K 2024-11-17T22:40:47,725 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/.tmp/C/eeea81bec4b74ba88da9f50897fefadd as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/eeea81bec4b74ba88da9f50897fefadd 2024-11-17T22:40:47,731 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/eeea81bec4b74ba88da9f50897fefadd, entries=150, sequenceid=369, filesize=12.0 K 2024-11-17T22:40:47,732 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3893ec8e69968d25a1df1a69563cc10b in 1694ms, sequenceid=369, compaction requested=true 2024-11-17T22:40:47,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/5a34bbb1f18840b3990e1d6b9c878a35, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/be9d15cc8e2f4f2ab6f396e188b5d6a6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9c16adc132c2443c9966f473b31af196, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/2e7ab5ad23a54136a365ae37730c673c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/a379540fe4264c61b0d027cfd8badae0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/b2619a91ecbd4f94b2f2119d76d1d2e8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/09ace82e77a748109d53c857b9b573e9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/dcb9a0481c664f41a30ddf6dd6b81c7e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/abcbfc24611945ae85900ecdc32d46fa, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/f6c673f33f5243d5af1a2123ab7658b2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/d94d73d4122347608613af34da7b3614, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7e0dc59fa6e443a2bfdb67fb09e8a01e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9ef13b608dc04a9783d2782a1ea3b1c4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c4a929e139bc441ba3caa7b2dde2c00f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/48c22566eb31489f8c87d5b5c6ce86c3, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c5ba1145812f4868ab65fbf8f50aa83f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/0bb8b1a84ea2421c8472deccdebc776e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/3a436e768dda42b6b1126313de4bb374, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/69635a37111c46938ebdc115acae9a00, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/187bcc1775d2451a9ba09922eacf4c7f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7c3a67196b0b4bcd84bbc7c1aafaf64a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/4dfb5e407e7a489fba491ca39bfa48e0] to archive 2024-11-17T22:40:47,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:40:47,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/5a34bbb1f18840b3990e1d6b9c878a35 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/5a34bbb1f18840b3990e1d6b9c878a35 2024-11-17T22:40:47,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/be9d15cc8e2f4f2ab6f396e188b5d6a6 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/be9d15cc8e2f4f2ab6f396e188b5d6a6 2024-11-17T22:40:47,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9c16adc132c2443c9966f473b31af196 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9c16adc132c2443c9966f473b31af196 2024-11-17T22:40:47,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/2e7ab5ad23a54136a365ae37730c673c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/2e7ab5ad23a54136a365ae37730c673c 2024-11-17T22:40:47,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/a379540fe4264c61b0d027cfd8badae0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/a379540fe4264c61b0d027cfd8badae0 2024-11-17T22:40:47,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/b2619a91ecbd4f94b2f2119d76d1d2e8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/b2619a91ecbd4f94b2f2119d76d1d2e8 2024-11-17T22:40:47,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/09ace82e77a748109d53c857b9b573e9 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/09ace82e77a748109d53c857b9b573e9 2024-11-17T22:40:47,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/dcb9a0481c664f41a30ddf6dd6b81c7e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/dcb9a0481c664f41a30ddf6dd6b81c7e 2024-11-17T22:40:47,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/abcbfc24611945ae85900ecdc32d46fa to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/abcbfc24611945ae85900ecdc32d46fa 2024-11-17T22:40:47,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/f6c673f33f5243d5af1a2123ab7658b2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/f6c673f33f5243d5af1a2123ab7658b2 2024-11-17T22:40:47,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/d94d73d4122347608613af34da7b3614 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/d94d73d4122347608613af34da7b3614 2024-11-17T22:40:47,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7e0dc59fa6e443a2bfdb67fb09e8a01e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7e0dc59fa6e443a2bfdb67fb09e8a01e 2024-11-17T22:40:47,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9ef13b608dc04a9783d2782a1ea3b1c4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/9ef13b608dc04a9783d2782a1ea3b1c4 2024-11-17T22:40:47,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c4a929e139bc441ba3caa7b2dde2c00f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c4a929e139bc441ba3caa7b2dde2c00f 2024-11-17T22:40:47,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/48c22566eb31489f8c87d5b5c6ce86c3 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/48c22566eb31489f8c87d5b5c6ce86c3 2024-11-17T22:40:47,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c5ba1145812f4868ab65fbf8f50aa83f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/c5ba1145812f4868ab65fbf8f50aa83f 2024-11-17T22:40:47,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/0bb8b1a84ea2421c8472deccdebc776e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/0bb8b1a84ea2421c8472deccdebc776e 2024-11-17T22:40:47,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/3a436e768dda42b6b1126313de4bb374 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/3a436e768dda42b6b1126313de4bb374 2024-11-17T22:40:47,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/69635a37111c46938ebdc115acae9a00 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/69635a37111c46938ebdc115acae9a00 2024-11-17T22:40:47,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/187bcc1775d2451a9ba09922eacf4c7f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/187bcc1775d2451a9ba09922eacf4c7f 2024-11-17T22:40:47,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7c3a67196b0b4bcd84bbc7c1aafaf64a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/7c3a67196b0b4bcd84bbc7c1aafaf64a 2024-11-17T22:40:47,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/4dfb5e407e7a489fba491ca39bfa48e0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/4dfb5e407e7a489fba491ca39bfa48e0 2024-11-17T22:40:47,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/68559556f69040429bef6be4cbab5812, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c375e18eeaaa4fefbdf0803acb4245dd, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/78ff77f9fc50462fb2b798f5eedee12e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/600eab9e4b464b21bc4cd61156884f41, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/43a1a6296668460f88759dc5af8c77b4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/70ebb6d4a08e4eb5be999e91d3f44a11, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/80cd30ba39014427b061225ad78acdca, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/0e4058ba403b4cb58aaaf5f37f2b527b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/d3f512fcaf1d4d3f95768c2513a18396, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/7b25a39096b341f288add0f3e6078ad5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/9d8d7f704a9d4e3d946d0773197bc7c5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/fb2e1338e892429b80eabdd0a36b78c3, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/db4bb70747cf4f22986205fbe5c90763, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6d4fb7c0821a45669745d1a8aba56c86, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f6b5c2b496f54dfcb48b3d421961659f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/b901375f60f04e74889595fc7d5b6b14, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f0257fbb86a04dc9b2797467c02c3c1f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c8c92e340b874e47a06641267c7b1c1f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/5ff9952def0345d3b4ed2bbe82c80a5a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6b343b7b543e4f059eec7fc2f028ef29, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/4625364b15fb49aca484485b955bb9c4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/37cf353006704656bbb7ea63d60d66f0] to archive 2024-11-17T22:40:47,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:40:47,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/68559556f69040429bef6be4cbab5812 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/68559556f69040429bef6be4cbab5812 2024-11-17T22:40:47,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c375e18eeaaa4fefbdf0803acb4245dd to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c375e18eeaaa4fefbdf0803acb4245dd 2024-11-17T22:40:47,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/78ff77f9fc50462fb2b798f5eedee12e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/78ff77f9fc50462fb2b798f5eedee12e 2024-11-17T22:40:47,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/600eab9e4b464b21bc4cd61156884f41 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/600eab9e4b464b21bc4cd61156884f41 2024-11-17T22:40:47,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/43a1a6296668460f88759dc5af8c77b4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/43a1a6296668460f88759dc5af8c77b4 2024-11-17T22:40:47,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/70ebb6d4a08e4eb5be999e91d3f44a11 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/70ebb6d4a08e4eb5be999e91d3f44a11 2024-11-17T22:40:47,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/80cd30ba39014427b061225ad78acdca to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/80cd30ba39014427b061225ad78acdca 2024-11-17T22:40:47,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/0e4058ba403b4cb58aaaf5f37f2b527b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/0e4058ba403b4cb58aaaf5f37f2b527b 2024-11-17T22:40:47,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/d3f512fcaf1d4d3f95768c2513a18396 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/d3f512fcaf1d4d3f95768c2513a18396 2024-11-17T22:40:47,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/7b25a39096b341f288add0f3e6078ad5 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/7b25a39096b341f288add0f3e6078ad5 2024-11-17T22:40:47,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/9d8d7f704a9d4e3d946d0773197bc7c5 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/9d8d7f704a9d4e3d946d0773197bc7c5 2024-11-17T22:40:47,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/fb2e1338e892429b80eabdd0a36b78c3 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/fb2e1338e892429b80eabdd0a36b78c3 2024-11-17T22:40:47,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/db4bb70747cf4f22986205fbe5c90763 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/db4bb70747cf4f22986205fbe5c90763 2024-11-17T22:40:47,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6d4fb7c0821a45669745d1a8aba56c86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6d4fb7c0821a45669745d1a8aba56c86 2024-11-17T22:40:47,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f6b5c2b496f54dfcb48b3d421961659f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f6b5c2b496f54dfcb48b3d421961659f 2024-11-17T22:40:47,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/b901375f60f04e74889595fc7d5b6b14 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/b901375f60f04e74889595fc7d5b6b14 2024-11-17T22:40:47,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f0257fbb86a04dc9b2797467c02c3c1f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f0257fbb86a04dc9b2797467c02c3c1f 2024-11-17T22:40:47,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c8c92e340b874e47a06641267c7b1c1f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c8c92e340b874e47a06641267c7b1c1f 2024-11-17T22:40:47,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/5ff9952def0345d3b4ed2bbe82c80a5a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/5ff9952def0345d3b4ed2bbe82c80a5a 2024-11-17T22:40:47,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6b343b7b543e4f059eec7fc2f028ef29 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/6b343b7b543e4f059eec7fc2f028ef29 2024-11-17T22:40:47,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/4625364b15fb49aca484485b955bb9c4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/4625364b15fb49aca484485b955bb9c4 2024-11-17T22:40:47,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/37cf353006704656bbb7ea63d60d66f0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/37cf353006704656bbb7ea63d60d66f0 2024-11-17T22:40:47,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3ce03cf6494f4dff937c83ad56c8b173, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2c79c992fda54e8ea5a8dbc0f2cbf296, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ffbf715b844f4447b760752e4d27a182, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3371461477d2461ebebf525ccf10f20b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a76059af63864384bd37a425f32d8f7c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/54da967fcadf41b89626ad30aab23ad2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/5d3daed4982b410e989b82783789a3d7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ec7427fd49a443dfb71325ce691cd52f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/abc7729aeb944ae69bbf09b6f52ede38, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/194c2603b3804b7eaff60f3c704281de, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2fa920313afc4096b289bd219d3664b3, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/d153f4cdf2064352a3edf11e2976e83d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c235c3a89d4e43e599a4dcf4367252b7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/de1af71ac2ae4a0a83da0701c3de8b28, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/4379f5c825bd478298533d5207bab9f6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/9004f924f58e4450865e2352b7e68ef7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2e0f35306c1e473fa20d0597a7a4529a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c65dd6c563444be2bb6ecea3f3f5f32f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/fda1a7fd42fe4d5996feaa19784fa103, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a3bde199b3f44937b17d891906e30a5a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/b811e8190b774e5c90e8450a98326950, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/bfe3f54374cf4a4785fd5e358d852056] to archive 2024-11-17T22:40:47,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:40:47,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3ce03cf6494f4dff937c83ad56c8b173 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3ce03cf6494f4dff937c83ad56c8b173 2024-11-17T22:40:47,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2c79c992fda54e8ea5a8dbc0f2cbf296 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2c79c992fda54e8ea5a8dbc0f2cbf296 2024-11-17T22:40:47,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ffbf715b844f4447b760752e4d27a182 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ffbf715b844f4447b760752e4d27a182 2024-11-17T22:40:47,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3371461477d2461ebebf525ccf10f20b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3371461477d2461ebebf525ccf10f20b 2024-11-17T22:40:47,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a76059af63864384bd37a425f32d8f7c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a76059af63864384bd37a425f32d8f7c 2024-11-17T22:40:47,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/54da967fcadf41b89626ad30aab23ad2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/54da967fcadf41b89626ad30aab23ad2 2024-11-17T22:40:47,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/5d3daed4982b410e989b82783789a3d7 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/5d3daed4982b410e989b82783789a3d7 2024-11-17T22:40:47,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ec7427fd49a443dfb71325ce691cd52f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ec7427fd49a443dfb71325ce691cd52f 2024-11-17T22:40:47,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/abc7729aeb944ae69bbf09b6f52ede38 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/abc7729aeb944ae69bbf09b6f52ede38 2024-11-17T22:40:47,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/194c2603b3804b7eaff60f3c704281de to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/194c2603b3804b7eaff60f3c704281de 2024-11-17T22:40:47,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2fa920313afc4096b289bd219d3664b3 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2fa920313afc4096b289bd219d3664b3 2024-11-17T22:40:47,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/d153f4cdf2064352a3edf11e2976e83d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/d153f4cdf2064352a3edf11e2976e83d 2024-11-17T22:40:47,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c235c3a89d4e43e599a4dcf4367252b7 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c235c3a89d4e43e599a4dcf4367252b7 2024-11-17T22:40:47,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/de1af71ac2ae4a0a83da0701c3de8b28 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/de1af71ac2ae4a0a83da0701c3de8b28 2024-11-17T22:40:47,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/4379f5c825bd478298533d5207bab9f6 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/4379f5c825bd478298533d5207bab9f6 2024-11-17T22:40:47,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/9004f924f58e4450865e2352b7e68ef7 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/9004f924f58e4450865e2352b7e68ef7 2024-11-17T22:40:47,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2e0f35306c1e473fa20d0597a7a4529a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/2e0f35306c1e473fa20d0597a7a4529a 2024-11-17T22:40:47,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c65dd6c563444be2bb6ecea3f3f5f32f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/c65dd6c563444be2bb6ecea3f3f5f32f 2024-11-17T22:40:47,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/fda1a7fd42fe4d5996feaa19784fa103 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/fda1a7fd42fe4d5996feaa19784fa103 2024-11-17T22:40:47,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a3bde199b3f44937b17d891906e30a5a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/a3bde199b3f44937b17d891906e30a5a 2024-11-17T22:40:47,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/b811e8190b774e5c90e8450a98326950 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/b811e8190b774e5c90e8450a98326950 2024-11-17T22:40:47,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/bfe3f54374cf4a4785fd5e358d852056 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/bfe3f54374cf4a4785fd5e358d852056 2024-11-17T22:40:47,825 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/recovered.edits/372.seqid, newMaxSeqId=372, maxSeqId=4 2024-11-17T22:40:47,826 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. 2024-11-17T22:40:47,826 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1676): Region close journal for 3893ec8e69968d25a1df1a69563cc10b: Waiting for close lock at 1731883246037Running coprocessor pre-close hooks at 1731883246037Disabling compacts and flushes for region at 1731883246037Disabling writes for close at 1731883246038 (+1 ms)Obtaining lock to block concurrent updates at 1731883246038Preparing flush snapshotting stores in 3893ec8e69968d25a1df1a69563cc10b at 1731883246038Finished memstore snapshotting TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731883246039 (+1 ms)Flushing stores of TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b. at 1731883246040 (+1 ms)Flushing 3893ec8e69968d25a1df1a69563cc10b/A: creating writer at 1731883246040Flushing mob file 3893ec8e69968d25a1df1a69563cc10b/A: appending metadata at 1731883246049 (+9 ms)Flushing mob file 3893ec8e69968d25a1df1a69563cc10b/A: closing flushed file at 1731883246049Flushing 3893ec8e69968d25a1df1a69563cc10b/A: appending metadata at 1731883246464 (+415 ms)Flushing 3893ec8e69968d25a1df1a69563cc10b/A: closing flushed file at 1731883246464Flushing 3893ec8e69968d25a1df1a69563cc10b/B: creating writer at 1731883246877 (+413 ms)Flushing 3893ec8e69968d25a1df1a69563cc10b/B: appending metadata at 1731883246882 (+5 ms)Flushing 3893ec8e69968d25a1df1a69563cc10b/B: closing flushed file at 1731883246882Flushing 3893ec8e69968d25a1df1a69563cc10b/C: creating writer at 1731883247292 (+410 ms)Flushing 3893ec8e69968d25a1df1a69563cc10b/C: appending metadata at 1731883247295 (+3 ms)Flushing 3893ec8e69968d25a1df1a69563cc10b/C: closing flushed file at 1731883247295Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29da9a71: reopening flushed file at 1731883247713 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e61ff13: reopening flushed file at 1731883247718 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65eab7f4: reopening flushed file at 1731883247724 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3893ec8e69968d25a1df1a69563cc10b in 1694ms, sequenceid=369, compaction requested=true at 1731883247732 (+8 ms)Writing region close event to WAL at 1731883247822 (+90 ms)Running coprocessor post-close hooks at 1731883247825 (+3 ms)Closed at 1731883247826 (+1 ms) 2024-11-17T22:40:47,827 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] handler.UnassignRegionHandler(157): Closed 3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:47,828 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=3893ec8e69968d25a1df1a69563cc10b, regionState=CLOSED 2024-11-17T22:40:47,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:40:47,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-17T22:40:47,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; CloseRegionProcedure 3893ec8e69968d25a1df1a69563cc10b, server=1a6e40b21a48,43011,1731883188895 in 1.9450 sec 2024-11-17T22:40:47,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-17T22:40:47,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3893ec8e69968d25a1df1a69563cc10b, UNASSIGN in 1.9480 sec 2024-11-17T22:40:47,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-17T22:40:47,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9510 sec 2024-11-17T22:40:47,834 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883247834"}]},"ts":"1731883247834"} 2024-11-17T22:40:47,835 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T22:40:47,835 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T22:40:47,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9600 sec 2024-11-17T22:40:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T22:40:48,027 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:48,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T22:40:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:48,030 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=70, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T22:40:48,030 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=70, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:48,033 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,035 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/recovered.edits] 2024-11-17T22:40:48,037 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/33a1d9e2a0374b8086a6f12f8af04723 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/33a1d9e2a0374b8086a6f12f8af04723 2024-11-17T22:40:48,038 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/78268150829e4af28c86dc649a9c8506 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/78268150829e4af28c86dc649a9c8506 2024-11-17T22:40:48,039 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/ff041b8cabd8484bb4d927e1ac2c46f2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/A/ff041b8cabd8484bb4d927e1ac2c46f2 2024-11-17T22:40:48,041 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/a8cd45847b144aa29f24356465e30812 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/a8cd45847b144aa29f24356465e30812 2024-11-17T22:40:48,042 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c6270c4177ae428cba75933d9537d70f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/c6270c4177ae428cba75933d9537d70f 2024-11-17T22:40:48,043 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f965de5326e54359911a6ac85929b5d5 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/B/f965de5326e54359911a6ac85929b5d5 2024-11-17T22:40:48,045 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3d70966c94ce4e4f8292c7b3db79dfde to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/3d70966c94ce4e4f8292c7b3db79dfde 2024-11-17T22:40:48,046 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ab8a4642b3cb40c2810b256160b0a512 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/ab8a4642b3cb40c2810b256160b0a512 2024-11-17T22:40:48,047 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/eeea81bec4b74ba88da9f50897fefadd to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/C/eeea81bec4b74ba88da9f50897fefadd 2024-11-17T22:40:48,050 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/recovered.edits/372.seqid to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b/recovered.edits/372.seqid 2024-11-17T22:40:48,050 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,050 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T22:40:48,051 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T22:40:48,052 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T22:40:48,056 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111711a15985e0ff44c6a0bba615ed17a6b8_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111711a15985e0ff44c6a0bba615ed17a6b8_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,057 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111720de093bd2a74b4088a43f78b6a662a6_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111720de093bd2a74b4088a43f78b6a662a6_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,058 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117246eb50ac353415baa579d142d02f289_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117246eb50ac353415baa579d142d02f289_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,060 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172672187955f343e6a9f11228e8dbb804_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172672187955f343e6a9f11228e8dbb804_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,061 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e252870b8a247dd9632175ee25008f8_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e252870b8a247dd9632175ee25008f8_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,062 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111739f7a2a0ccdf4400ac65b7161b6247df_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111739f7a2a0ccdf4400ac65b7161b6247df_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,064 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173c32589684ef4b6b80bc35475bffaeb2_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173c32589684ef4b6b80bc35475bffaeb2_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,066 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111742dfe5271a2e4292bba522bf761b30b8_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111742dfe5271a2e4292bba522bf761b30b8_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,068 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174fd71970a55c4a8bab30af6fa66ee49b_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174fd71970a55c4a8bab30af6fa66ee49b_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,069 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111769d6bf2085fb4ed49ec539f3111d266d_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111769d6bf2085fb4ed49ec539f3111d266d_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,070 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176fd971e6a8ec4bfd8d2747ecf7407d2d_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176fd971e6a8ec4bfd8d2747ecf7407d2d_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,071 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111792f8e4fc3a9f4767ae25c50b852fec96_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111792f8e4fc3a9f4767ae25c50b852fec96_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,072 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ae85028ab17c447bb787e9e0f0164d83_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ae85028ab17c447bb787e9e0f0164d83_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,073 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b8b58d1fa67d42968b7ededd3a9d5b4e_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b8b58d1fa67d42968b7ededd3a9d5b4e_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,074 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b9eb65d0091d4c6fa5baf482effda9c8_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b9eb65d0091d4c6fa5baf482effda9c8_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,075 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c22ac9542412464c8a7008cabd070bfe_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c22ac9542412464c8a7008cabd070bfe_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,075 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c772910f6896461d8998c9b34475e7e4_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c772910f6896461d8998c9b34475e7e4_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,076 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d3020e49991e401dacd900d12e0f1b1e_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d3020e49991e401dacd900d12e0f1b1e_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,077 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117de1083874cd84c85a58533ede738bfef_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117de1083874cd84c85a58533ede738bfef_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,078 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e693aa50dbaa4ca28e7a772251f0f163_3893ec8e69968d25a1df1a69563cc10b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e693aa50dbaa4ca28e7a772251f0f163_3893ec8e69968d25a1df1a69563cc10b 2024-11-17T22:40:48,078 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T22:40:48,080 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=70, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:48,082 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T22:40:48,084 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T22:40:48,085 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=70, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:48,085 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T22:40:48,085 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731883248085"}]},"ts":"9223372036854775807"} 2024-11-17T22:40:48,087 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T22:40:48,087 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3893ec8e69968d25a1df1a69563cc10b, NAME => 'TestAcidGuarantees,,1731883219019.3893ec8e69968d25a1df1a69563cc10b.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T22:40:48,087 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T22:40:48,087 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731883248087"}]},"ts":"9223372036854775807"} 2024-11-17T22:40:48,089 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T22:40:48,090 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=70, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:48,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-17T22:40:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T22:40:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T22:40:48,138 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:48,146 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240 (was 238) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-225167225_22 at /127.0.0.1:55504 [Waiting for operation #581] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/cluster_de41a222-fb27-5a57-9fde-672530362f70/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-39089393_22 at /127.0.0.1:55634 [Waiting for operation #573] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/cluster_de41a222-fb27-5a57-9fde-672530362f70/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=399 (was 371) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4699 (was 4366) - AvailableMemoryMB LEAK? - 2024-11-17T22:40:48,153 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=399, ProcessCount=11, AvailableMemoryMB=4699 2024-11-17T22:40:48,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T22:40:48,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T22:40:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=71, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T22:40:48,156 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T22:40:48,157 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:48,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 71 2024-11-17T22:40:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-11-17T22:40:48,158 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T22:40:48,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742047_1223 (size=959) 2024-11-17T22:40:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-11-17T22:40:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-11-17T22:40:48,567 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71 2024-11-17T22:40:48,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742048_1224 (size=53) 2024-11-17T22:40:48,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T22:40:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-11-17T22:40:48,974 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:40:48,975 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ad1461c99ce1acccd0c930b472c9a7c1, disabling compactions & flushes 2024-11-17T22:40:48,975 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:48,975 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:48,975 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. after waiting 0 ms 2024-11-17T22:40:48,975 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:48,975 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:48,975 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ad1461c99ce1acccd0c930b472c9a7c1: Waiting for close lock at 1731883248974Disabling compacts and flushes for region at 1731883248974Disabling writes for close at 1731883248975 (+1 ms)Writing region close event to WAL at 1731883248975Closed at 1731883248975 2024-11-17T22:40:48,978 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T22:40:48,979 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731883248978"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731883248978"}]},"ts":"1731883248978"} 2024-11-17T22:40:48,982 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T22:40:48,983 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T22:40:48,983 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883248983"}]},"ts":"1731883248983"} 2024-11-17T22:40:48,985 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T22:40:48,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1461c99ce1acccd0c930b472c9a7c1, ASSIGN}] 2024-11-17T22:40:48,986 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1461c99ce1acccd0c930b472c9a7c1, ASSIGN 2024-11-17T22:40:48,986 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1461c99ce1acccd0c930b472c9a7c1, ASSIGN; state=OFFLINE, location=1a6e40b21a48,43011,1731883188895; forceNewPlan=false, retain=false 2024-11-17T22:40:49,137 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=ad1461c99ce1acccd0c930b472c9a7c1, regionState=OPENING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:49,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1461c99ce1acccd0c930b472c9a7c1, ASSIGN because future has completed 2024-11-17T22:40:49,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:40:49,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-11-17T22:40:49,297 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:49,297 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} 2024-11-17T22:40:49,298 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,298 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:40:49,298 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,298 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,299 INFO [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,300 INFO [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad1461c99ce1acccd0c930b472c9a7c1 columnFamilyName A 2024-11-17T22:40:49,300 DEBUG [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:49,300 INFO [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] regionserver.HStore(327): Store=ad1461c99ce1acccd0c930b472c9a7c1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:40:49,300 INFO [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,301 INFO [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad1461c99ce1acccd0c930b472c9a7c1 columnFamilyName B 2024-11-17T22:40:49,301 DEBUG [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:49,302 INFO [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] regionserver.HStore(327): Store=ad1461c99ce1acccd0c930b472c9a7c1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:40:49,302 INFO [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,303 INFO [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad1461c99ce1acccd0c930b472c9a7c1 columnFamilyName C 2024-11-17T22:40:49,303 DEBUG [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:40:49,303 INFO [StoreOpener-ad1461c99ce1acccd0c930b472c9a7c1-1 {}] regionserver.HStore(327): Store=ad1461c99ce1acccd0c930b472c9a7c1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:40:49,303 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,304 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,304 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,305 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,305 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,305 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T22:40:49,306 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,308 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T22:40:49,309 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened ad1461c99ce1acccd0c930b472c9a7c1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61390157, jitterRate=-0.08521537482738495}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T22:40:49,309 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:49,309 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for ad1461c99ce1acccd0c930b472c9a7c1: Running coprocessor pre-open hook at 1731883249298Writing region info on filesystem at 1731883249298Initializing all the Stores at 1731883249299 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883249299Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883249299Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883249299Cleaning up temporary data from old regions at 1731883249305 (+6 ms)Running coprocessor post-open hooks at 1731883249309 (+4 ms)Region opened successfully at 1731883249309 2024-11-17T22:40:49,310 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., pid=73, masterSystemTime=1731883249294 2024-11-17T22:40:49,312 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:49,312 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:49,312 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=ad1461c99ce1acccd0c930b472c9a7c1, regionState=OPEN, openSeqNum=2, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:49,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:40:49,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-17T22:40:49,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 in 173 msec 2024-11-17T22:40:49,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71 2024-11-17T22:40:49,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1461c99ce1acccd0c930b472c9a7c1, ASSIGN in 331 msec 2024-11-17T22:40:49,318 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T22:40:49,318 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883249318"}]},"ts":"1731883249318"} 2024-11-17T22:40:49,319 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T22:40:49,320 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T22:40:49,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-17T22:40:50,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-11-17T22:40:50,307 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:50,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3260768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:50,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:50,309 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65c72723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:50,311 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4832e03b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,312 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:50,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:50,316 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T22:40:50,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a6e40b21a48,45671,1731883188234 2024-11-17T22:40:50,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76ae3565 2024-11-17T22:40:50,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T22:40:50,318 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T22:40:50,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555b0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:50,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:50,320 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e4ab7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:50,321 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f95d5c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:50,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:50,323 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a76607, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:50,324 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6311abb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:50,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:50,326 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213336de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:50,327 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:50,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f73370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:50,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:50,329 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088aa65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:50,330 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d192f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:50,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:50,332 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720588f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:50,334 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:50,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f09fd8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:50,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:50,337 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7e1c3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:50,338 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:50,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc3348f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,339 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:50,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:50,340 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be65df4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:50,341 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@539d75d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:50,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:50,343 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755924be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:50,345 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:50,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4389e259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,346 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:50,346 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:50,347 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:50,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:50,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:50,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d458b9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:50,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:50,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:50,348 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4fe37a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,351 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:40:50,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:40:50,351 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:40:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:40:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:40:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba1a484, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:40:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:40:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:40:50,353 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:40:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ee904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:40:50,354 DEBUG [Thread-1063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:50,354 DEBUG [Thread-1065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:50,354 DEBUG [Thread-1050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:50,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:50,354 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:50,355 DEBUG [Thread-1061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:50,355 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:50,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-17T22:40:50,356 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:50,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:50,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:50,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:50,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:50,357 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:50,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T22:40:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:50,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:50,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:50,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:50,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:40:50,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:40:50,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:50,359 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:40:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:40:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:40:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:40:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:40:50,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:40:50,362 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:50,362 DEBUG [Thread-1067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:40:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:50,363 DEBUG [Thread-1067 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:40:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:50,363 DEBUG [Thread-1067 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:40:50,363 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:50,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:50,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:50,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:40:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:40:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:40:50,372 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:50,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:40:50,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:50,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/e4cf5bbfdc234ca99572c427b970940c is 50, key is test_row_0/A:col10/1731883250369/Put/seqid=0 2024-11-17T22:40:50,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883310403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883310405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883310406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883310411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883310423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742049_1225 (size=12001) 2024-11-17T22:40:50,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/e4cf5bbfdc234ca99572c427b970940c 2024-11-17T22:40:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T22:40:50,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/5dceec0d506e46a88d7823468d147327 is 50, key is test_row_0/B:col10/1731883250369/Put/seqid=0 2024-11-17T22:40:50,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T22:40:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:50,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742050_1226 (size=12001) 2024-11-17T22:40:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883310517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883310517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883310527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883310527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883310536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T22:40:50,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:50,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:50,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:50,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T22:40:50,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883310727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883310727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883310736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883310736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883310747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:50,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T22:40:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:50,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/5dceec0d506e46a88d7823468d147327 2024-11-17T22:40:50,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/7830e5e49a424a009d0142f94e8cb940 is 50, key is test_row_0/C:col10/1731883250369/Put/seqid=0 2024-11-17T22:40:50,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742051_1227 (size=12001) 2024-11-17T22:40:50,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T22:40:50,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:50,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:50,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:50,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T22:40:51,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883311036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883311036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883311047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883311047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883311057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T22:40:51,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:51,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:51,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:51,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:51,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:51,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T22:40:51,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:51,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:51,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:51,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:51,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:51,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/7830e5e49a424a009d0142f94e8cb940 2024-11-17T22:40:51,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/e4cf5bbfdc234ca99572c427b970940c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/e4cf5bbfdc234ca99572c427b970940c 2024-11-17T22:40:51,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/e4cf5bbfdc234ca99572c427b970940c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T22:40:51,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/5dceec0d506e46a88d7823468d147327 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5dceec0d506e46a88d7823468d147327 2024-11-17T22:40:51,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5dceec0d506e46a88d7823468d147327, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T22:40:51,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/7830e5e49a424a009d0142f94e8cb940 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7830e5e49a424a009d0142f94e8cb940 2024-11-17T22:40:51,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7830e5e49a424a009d0142f94e8cb940, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T22:40:51,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for ad1461c99ce1acccd0c930b472c9a7c1 in 1011ms, sequenceid=13, compaction requested=false 2024-11-17T22:40:51,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:51,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T22:40:51,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:51,443 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:40:51,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ce47feb310354e9a8a616809e98d075f is 50, key is test_row_0/A:col10/1731883250402/Put/seqid=0 2024-11-17T22:40:51,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742052_1228 (size=12001) 2024-11-17T22:40:51,494 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ce47feb310354e9a8a616809e98d075f 2024-11-17T22:40:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T22:40:51,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/1715ffabebb449239214867eab71ab9c is 50, key is test_row_0/B:col10/1731883250402/Put/seqid=0 2024-11-17T22:40:51,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742053_1229 (size=12001) 2024-11-17T22:40:51,542 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/1715ffabebb449239214867eab71ab9c 2024-11-17T22:40:51,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:51,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883311562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883311563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883311563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883311564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883311576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/7917c69aae16441da70ee5561f0557f2 is 50, key is test_row_0/C:col10/1731883250402/Put/seqid=0 2024-11-17T22:40:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742054_1230 (size=12001) 2024-11-17T22:40:51,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883311677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883311677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883311678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883311678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883311887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883311887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883311887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883311887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,020 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/7917c69aae16441da70ee5561f0557f2 2024-11-17T22:40:52,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ce47feb310354e9a8a616809e98d075f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ce47feb310354e9a8a616809e98d075f 2024-11-17T22:40:52,043 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ce47feb310354e9a8a616809e98d075f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T22:40:52,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/1715ffabebb449239214867eab71ab9c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/1715ffabebb449239214867eab71ab9c 2024-11-17T22:40:52,054 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/1715ffabebb449239214867eab71ab9c, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T22:40:52,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/7917c69aae16441da70ee5561f0557f2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7917c69aae16441da70ee5561f0557f2 2024-11-17T22:40:52,066 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7917c69aae16441da70ee5561f0557f2, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T22:40:52,067 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ad1461c99ce1acccd0c930b472c9a7c1 in 624ms, sequenceid=38, compaction requested=false 2024-11-17T22:40:52,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-17T22:40:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-17T22:40:52,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-17T22:40:52,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7070 sec 2024-11-17T22:40:52,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.7220 sec 2024-11-17T22:40:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:52,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:52,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:40:52,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/d3831fcdd5734ddf9b5b8cd3e8201f28 is 50, key is test_row_0/A:col10/1731883251556/Put/seqid=0 2024-11-17T22:40:52,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883312242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883312242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883312243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883312243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742055_1231 (size=12001) 2024-11-17T22:40:52,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/d3831fcdd5734ddf9b5b8cd3e8201f28 2024-11-17T22:40:52,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/139a95b0b11c4beb8ead5aa5d7a3a0d8 is 50, key is test_row_0/B:col10/1731883251556/Put/seqid=0 2024-11-17T22:40:52,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742056_1232 (size=12001) 2024-11-17T22:40:52,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/139a95b0b11c4beb8ead5aa5d7a3a0d8 2024-11-17T22:40:52,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/16a14ee51d074a70891e85f91b6fed5c is 50, key is test_row_0/C:col10/1731883251556/Put/seqid=0 2024-11-17T22:40:52,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742057_1233 (size=12001) 2024-11-17T22:40:52,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/16a14ee51d074a70891e85f91b6fed5c 2024-11-17T22:40:52,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/d3831fcdd5734ddf9b5b8cd3e8201f28 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d3831fcdd5734ddf9b5b8cd3e8201f28 2024-11-17T22:40:52,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d3831fcdd5734ddf9b5b8cd3e8201f28, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T22:40:52,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/139a95b0b11c4beb8ead5aa5d7a3a0d8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/139a95b0b11c4beb8ead5aa5d7a3a0d8 2024-11-17T22:40:52,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/139a95b0b11c4beb8ead5aa5d7a3a0d8, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T22:40:52,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/16a14ee51d074a70891e85f91b6fed5c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/16a14ee51d074a70891e85f91b6fed5c 2024-11-17T22:40:52,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883312357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883312357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883312357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883312357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/16a14ee51d074a70891e85f91b6fed5c, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T22:40:52,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ad1461c99ce1acccd0c930b472c9a7c1 in 174ms, sequenceid=51, compaction requested=true 2024-11-17T22:40:52,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:52,374 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:52,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:52,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:52,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:52,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:52,375 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:52,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:52,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:52,376 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:52,376 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/A is initiating minor compaction (all files) 2024-11-17T22:40:52,376 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/A in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:52,376 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/e4cf5bbfdc234ca99572c427b970940c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ce47feb310354e9a8a616809e98d075f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d3831fcdd5734ddf9b5b8cd3e8201f28] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=35.2 K 2024-11-17T22:40:52,376 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:52,376 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/B is initiating minor compaction (all files) 2024-11-17T22:40:52,376 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/B in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:52,376 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5dceec0d506e46a88d7823468d147327, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/1715ffabebb449239214867eab71ab9c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/139a95b0b11c4beb8ead5aa5d7a3a0d8] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=35.2 K 2024-11-17T22:40:52,378 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dceec0d506e46a88d7823468d147327, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731883250362 2024-11-17T22:40:52,378 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4cf5bbfdc234ca99572c427b970940c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731883250362 2024-11-17T22:40:52,379 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 1715ffabebb449239214867eab71ab9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731883250402 2024-11-17T22:40:52,379 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce47feb310354e9a8a616809e98d075f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731883250402 2024-11-17T22:40:52,379 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 139a95b0b11c4beb8ead5aa5d7a3a0d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731883251556 2024-11-17T22:40:52,381 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3831fcdd5734ddf9b5b8cd3e8201f28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731883251556 2024-11-17T22:40:52,400 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#B#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:52,401 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/3523d91eb864401ca35e22a3ed46de25 is 50, key is test_row_0/B:col10/1731883251556/Put/seqid=0 2024-11-17T22:40:52,401 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#A#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:52,401 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/8af6d94466aa4e9f9dda01258a2ce1ef is 50, key is test_row_0/A:col10/1731883251556/Put/seqid=0 2024-11-17T22:40:52,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742058_1234 (size=12104) 2024-11-17T22:40:52,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742059_1235 (size=12104) 2024-11-17T22:40:52,446 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/8af6d94466aa4e9f9dda01258a2ce1ef as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/8af6d94466aa4e9f9dda01258a2ce1ef 2024-11-17T22:40:52,453 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/A of ad1461c99ce1acccd0c930b472c9a7c1 into 8af6d94466aa4e9f9dda01258a2ce1ef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:52,453 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:52,453 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/A, priority=13, startTime=1731883252374; duration=0sec 2024-11-17T22:40:52,453 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:52,453 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:A 2024-11-17T22:40:52,454 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:52,455 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:52,455 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/C is initiating minor compaction (all files) 2024-11-17T22:40:52,455 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/C in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:52,455 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7830e5e49a424a009d0142f94e8cb940, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7917c69aae16441da70ee5561f0557f2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/16a14ee51d074a70891e85f91b6fed5c] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=35.2 K 2024-11-17T22:40:52,456 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7830e5e49a424a009d0142f94e8cb940, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731883250362 2024-11-17T22:40:52,456 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7917c69aae16441da70ee5561f0557f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731883250402 2024-11-17T22:40:52,456 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16a14ee51d074a70891e85f91b6fed5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731883251556 2024-11-17T22:40:52,464 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/3523d91eb864401ca35e22a3ed46de25 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/3523d91eb864401ca35e22a3ed46de25 2024-11-17T22:40:52,471 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#C#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:52,471 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/9ec0334c42f446b6a4338f391af11aaf is 50, key is test_row_0/C:col10/1731883251556/Put/seqid=0 2024-11-17T22:40:52,475 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/B of ad1461c99ce1acccd0c930b472c9a7c1 into 3523d91eb864401ca35e22a3ed46de25(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:52,475 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:52,475 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/B, priority=13, startTime=1731883252374; duration=0sec 2024-11-17T22:40:52,475 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:52,475 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:B 2024-11-17T22:40:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T22:40:52,517 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:52,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-17T22:40:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T22:40:52,520 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:52,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:52,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:52,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742060_1236 (size=12104) 2024-11-17T22:40:52,540 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/9ec0334c42f446b6a4338f391af11aaf as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/9ec0334c42f446b6a4338f391af11aaf 2024-11-17T22:40:52,548 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/C of ad1461c99ce1acccd0c930b472c9a7c1 into 9ec0334c42f446b6a4338f391af11aaf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:52,548 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:52,549 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/C, priority=13, startTime=1731883252375; duration=0sec 2024-11-17T22:40:52,549 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:52,549 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:C 2024-11-17T22:40:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:52,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:40:52,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ffcaf29af53a4c499ebd3a73f8b4cfc7 is 50, key is test_row_0/A:col10/1731883252569/Put/seqid=0 2024-11-17T22:40:52,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742061_1237 (size=16681) 2024-11-17T22:40:52,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ffcaf29af53a4c499ebd3a73f8b4cfc7 2024-11-17T22:40:52,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883312584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883312587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883312587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883312590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b2f75a89d5dc492bb1133c3a5e7ae8f7 is 50, key is test_row_0/B:col10/1731883252569/Put/seqid=0 2024-11-17T22:40:52,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883312601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,605 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in ad1461c99ce1acccd0c930b472c9a7c1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:40:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742062_1238 (size=12001) 2024-11-17T22:40:52,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b2f75a89d5dc492bb1133c3a5e7ae8f7 2024-11-17T22:40:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T22:40:52,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/2053ef2925bb4b5dadd5ed76f01b0f6b is 50, key is test_row_0/C:col10/1731883252569/Put/seqid=0 2024-11-17T22:40:52,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742063_1239 (size=12001) 2024-11-17T22:40:52,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/2053ef2925bb4b5dadd5ed76f01b0f6b 2024-11-17T22:40:52,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ffcaf29af53a4c499ebd3a73f8b4cfc7 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ffcaf29af53a4c499ebd3a73f8b4cfc7 2024-11-17T22:40:52,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ffcaf29af53a4c499ebd3a73f8b4cfc7, entries=250, sequenceid=79, filesize=16.3 K 2024-11-17T22:40:52,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b2f75a89d5dc492bb1133c3a5e7ae8f7 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b2f75a89d5dc492bb1133c3a5e7ae8f7 2024-11-17T22:40:52,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b2f75a89d5dc492bb1133c3a5e7ae8f7, entries=150, sequenceid=79, filesize=11.7 K 2024-11-17T22:40:52,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T22:40:52,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:52,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:52,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:52,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:52,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:52,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/2053ef2925bb4b5dadd5ed76f01b0f6b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2053ef2925bb4b5dadd5ed76f01b0f6b 2024-11-17T22:40:52,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2053ef2925bb4b5dadd5ed76f01b0f6b, entries=150, sequenceid=79, filesize=11.7 K 2024-11-17T22:40:52,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ad1461c99ce1acccd0c930b472c9a7c1 in 116ms, sequenceid=79, compaction requested=false 2024-11-17T22:40:52,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:52,700 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:52,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:40:52,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/49cc2c9e167a43abae9cc8a12a9d4d7e is 50, key is test_row_0/A:col10/1731883252587/Put/seqid=0 2024-11-17T22:40:52,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742064_1240 (size=16681) 2024-11-17T22:40:52,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/49cc2c9e167a43abae9cc8a12a9d4d7e 2024-11-17T22:40:52,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/6cde69906bb4448191abafee40f5bd2f is 50, key is test_row_0/B:col10/1731883252587/Put/seqid=0 2024-11-17T22:40:52,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883312753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883312753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883312754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883312755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742065_1241 (size=12001) 2024-11-17T22:40:52,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/6cde69906bb4448191abafee40f5bd2f 2024-11-17T22:40:52,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/3da659b8923a4e52a2322453a2abf8e1 is 50, key is test_row_0/C:col10/1731883252587/Put/seqid=0 2024-11-17T22:40:52,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T22:40:52,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:52,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:52,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:52,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:52,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T22:40:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:52,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883312867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883312867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883312867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883312867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:52,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742066_1242 (size=12001) 2024-11-17T22:40:52,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/3da659b8923a4e52a2322453a2abf8e1 2024-11-17T22:40:52,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/49cc2c9e167a43abae9cc8a12a9d4d7e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/49cc2c9e167a43abae9cc8a12a9d4d7e 2024-11-17T22:40:52,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/49cc2c9e167a43abae9cc8a12a9d4d7e, entries=250, sequenceid=93, filesize=16.3 K 2024-11-17T22:40:52,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/6cde69906bb4448191abafee40f5bd2f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/6cde69906bb4448191abafee40f5bd2f 2024-11-17T22:40:52,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/6cde69906bb4448191abafee40f5bd2f, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T22:40:52,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/3da659b8923a4e52a2322453a2abf8e1 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3da659b8923a4e52a2322453a2abf8e1 2024-11-17T22:40:52,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3da659b8923a4e52a2322453a2abf8e1, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T22:40:52,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ad1461c99ce1acccd0c930b472c9a7c1 in 240ms, sequenceid=93, compaction requested=true 2024-11-17T22:40:52,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:52,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:52,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:52,940 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:52,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:52,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:52,941 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:52,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:52,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:52,942 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:52,942 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45466 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:52,942 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/B is initiating minor compaction (all files) 2024-11-17T22:40:52,942 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/A is initiating minor compaction (all files) 2024-11-17T22:40:52,942 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/B in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:52,942 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/A in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:52,942 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/8af6d94466aa4e9f9dda01258a2ce1ef, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ffcaf29af53a4c499ebd3a73f8b4cfc7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/49cc2c9e167a43abae9cc8a12a9d4d7e] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=44.4 K 2024-11-17T22:40:52,942 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/3523d91eb864401ca35e22a3ed46de25, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b2f75a89d5dc492bb1133c3a5e7ae8f7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/6cde69906bb4448191abafee40f5bd2f] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=35.3 K 2024-11-17T22:40:52,943 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 3523d91eb864401ca35e22a3ed46de25, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731883251556 2024-11-17T22:40:52,943 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8af6d94466aa4e9f9dda01258a2ce1ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731883251556 2024-11-17T22:40:52,943 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffcaf29af53a4c499ebd3a73f8b4cfc7, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731883252231 2024-11-17T22:40:52,943 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f75a89d5dc492bb1133c3a5e7ae8f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731883252231 2024-11-17T22:40:52,943 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49cc2c9e167a43abae9cc8a12a9d4d7e, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731883252583 2024-11-17T22:40:52,943 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cde69906bb4448191abafee40f5bd2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731883252587 2024-11-17T22:40:52,964 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#A#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:52,964 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/3a7f904b78b0498580545f8c338c74b9 is 50, key is test_row_0/A:col10/1731883252587/Put/seqid=0 2024-11-17T22:40:52,967 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:52,967 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/2dfd6957290e42b8ae15a0eaefec807e is 50, key is test_row_0/B:col10/1731883252587/Put/seqid=0 2024-11-17T22:40:53,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T22:40:53,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:53,009 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T22:40:53,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742067_1243 (size=12207) 2024-11-17T22:40:53,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/daad925cd59f4cdfb2a0eb4d10ba5f05 is 50, key is test_row_0/A:col10/1731883252747/Put/seqid=0 2024-11-17T22:40:53,058 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/2dfd6957290e42b8ae15a0eaefec807e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2dfd6957290e42b8ae15a0eaefec807e 2024-11-17T22:40:53,071 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/B of ad1461c99ce1acccd0c930b472c9a7c1 into 2dfd6957290e42b8ae15a0eaefec807e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:53,071 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:53,072 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/B, priority=13, startTime=1731883252940; duration=0sec 2024-11-17T22:40:53,072 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:53,072 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:B 2024-11-17T22:40:53,072 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:53,075 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:53,075 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/C is initiating minor compaction (all files) 2024-11-17T22:40:53,075 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/C in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:53,076 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/9ec0334c42f446b6a4338f391af11aaf, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2053ef2925bb4b5dadd5ed76f01b0f6b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3da659b8923a4e52a2322453a2abf8e1] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=35.3 K 2024-11-17T22:40:53,077 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ec0334c42f446b6a4338f391af11aaf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731883251556 2024-11-17T22:40:53,078 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 2053ef2925bb4b5dadd5ed76f01b0f6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731883252231 2024-11-17T22:40:53,078 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da659b8923a4e52a2322453a2abf8e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731883252587 2024-11-17T22:40:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:53,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:53,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742068_1244 (size=12207) 2024-11-17T22:40:53,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742069_1245 (size=12001) 2024-11-17T22:40:53,091 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/daad925cd59f4cdfb2a0eb4d10ba5f05 2024-11-17T22:40:53,097 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:53,098 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/ce7ea50943214167a3508be4efc159d2 is 50, key is test_row_0/C:col10/1731883252587/Put/seqid=0 2024-11-17T22:40:53,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/4c4c03ae5941411aa6091193aebb7b9b is 50, key is test_row_0/B:col10/1731883252747/Put/seqid=0 2024-11-17T22:40:53,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883313113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883313113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883313115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742071_1247 (size=12001) 2024-11-17T22:40:53,124 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/4c4c03ae5941411aa6091193aebb7b9b 2024-11-17T22:40:53,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883313122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742070_1246 (size=12207) 2024-11-17T22:40:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T22:40:53,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/fa1b1445d2c544ccb2bd6129d68c2c52 is 50, key is test_row_0/C:col10/1731883252747/Put/seqid=0 2024-11-17T22:40:53,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742072_1248 (size=12001) 2024-11-17T22:40:53,202 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/fa1b1445d2c544ccb2bd6129d68c2c52 2024-11-17T22:40:53,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/daad925cd59f4cdfb2a0eb4d10ba5f05 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/daad925cd59f4cdfb2a0eb4d10ba5f05 2024-11-17T22:40:53,214 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/daad925cd59f4cdfb2a0eb4d10ba5f05, entries=150, sequenceid=115, filesize=11.7 K 2024-11-17T22:40:53,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/4c4c03ae5941411aa6091193aebb7b9b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4c4c03ae5941411aa6091193aebb7b9b 2024-11-17T22:40:53,221 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4c4c03ae5941411aa6091193aebb7b9b, entries=150, sequenceid=115, filesize=11.7 K 2024-11-17T22:40:53,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/fa1b1445d2c544ccb2bd6129d68c2c52 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/fa1b1445d2c544ccb2bd6129d68c2c52 2024-11-17T22:40:53,229 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/fa1b1445d2c544ccb2bd6129d68c2c52, entries=150, sequenceid=115, filesize=11.7 K 2024-11-17T22:40:53,230 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ad1461c99ce1acccd0c930b472c9a7c1 in 221ms, sequenceid=115, compaction requested=false 2024-11-17T22:40:53,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:53,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:53,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-17T22:40:53,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-17T22:40:53,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-17T22:40:53,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 712 msec 2024-11-17T22:40:53,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:53,239 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:53,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T22:40:53,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 720 msec 2024-11-17T22:40:53,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/51b8604e80e8479181b83fb6cfdbe56a is 50, key is test_row_0/A:col10/1731883253113/Put/seqid=0 2024-11-17T22:40:53,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742073_1249 (size=14491) 2024-11-17T22:40:53,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883313283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883313285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/51b8604e80e8479181b83fb6cfdbe56a 2024-11-17T22:40:53,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883313287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883313290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/d25964c98cc747b8bff060f579a5c592 is 50, key is test_row_0/B:col10/1731883253113/Put/seqid=0 2024-11-17T22:40:53,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742074_1250 (size=12101) 2024-11-17T22:40:53,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883313397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883313397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,407 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T22:40:53,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883313408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883313408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,492 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/3a7f904b78b0498580545f8c338c74b9 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/3a7f904b78b0498580545f8c338c74b9 2024-11-17T22:40:53,501 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/A of ad1461c99ce1acccd0c930b472c9a7c1 into 3a7f904b78b0498580545f8c338c74b9(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:53,502 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:53,502 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/A, priority=13, startTime=1731883252940; duration=0sec 2024-11-17T22:40:53,502 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:53,502 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:A 2024-11-17T22:40:53,569 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/ce7ea50943214167a3508be4efc159d2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/ce7ea50943214167a3508be4efc159d2 2024-11-17T22:40:53,575 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/C of ad1461c99ce1acccd0c930b472c9a7c1 into ce7ea50943214167a3508be4efc159d2(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:53,575 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:53,575 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/C, priority=13, startTime=1731883252941; duration=0sec 2024-11-17T22:40:53,575 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:53,575 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:C 2024-11-17T22:40:53,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883313607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883313607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883313633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883313634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T22:40:53,677 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:53,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-17T22:40:53,680 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T22:40:53,682 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:53,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:53,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/d25964c98cc747b8bff060f579a5c592 2024-11-17T22:40:53,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/0586271406c942c7b5912ceefb3c3741 is 50, key is test_row_0/C:col10/1731883253113/Put/seqid=0 2024-11-17T22:40:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T22:40:53,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742075_1251 (size=12101) 2024-11-17T22:40:53,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/0586271406c942c7b5912ceefb3c3741 2024-11-17T22:40:53,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/51b8604e80e8479181b83fb6cfdbe56a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/51b8604e80e8479181b83fb6cfdbe56a 2024-11-17T22:40:53,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/51b8604e80e8479181b83fb6cfdbe56a, entries=200, sequenceid=133, filesize=14.2 K 2024-11-17T22:40:53,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/d25964c98cc747b8bff060f579a5c592 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/d25964c98cc747b8bff060f579a5c592 2024-11-17T22:40:53,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/d25964c98cc747b8bff060f579a5c592, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T22:40:53,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/0586271406c942c7b5912ceefb3c3741 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0586271406c942c7b5912ceefb3c3741 2024-11-17T22:40:53,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0586271406c942c7b5912ceefb3c3741, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T22:40:53,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for ad1461c99ce1acccd0c930b472c9a7c1 in 595ms, sequenceid=133, compaction requested=true 2024-11-17T22:40:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:53,834 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:53,834 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:53,835 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:53,836 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/B is initiating minor compaction (all files) 2024-11-17T22:40:53,836 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/B in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:53,836 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2dfd6957290e42b8ae15a0eaefec807e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4c4c03ae5941411aa6091193aebb7b9b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/d25964c98cc747b8bff060f579a5c592] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=35.5 K 2024-11-17T22:40:53,836 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:53,836 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/A is initiating minor compaction (all files) 2024-11-17T22:40:53,836 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/A in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:53,836 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/3a7f904b78b0498580545f8c338c74b9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/daad925cd59f4cdfb2a0eb4d10ba5f05, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/51b8604e80e8479181b83fb6cfdbe56a] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=37.8 K 2024-11-17T22:40:53,837 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a7f904b78b0498580545f8c338c74b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731883252587 2024-11-17T22:40:53,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T22:40:53,837 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dfd6957290e42b8ae15a0eaefec807e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731883252587 2024-11-17T22:40:53,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:53,837 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-17T22:40:53,838 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting daad925cd59f4cdfb2a0eb4d10ba5f05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731883252747 2024-11-17T22:40:53,838 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c4c03ae5941411aa6091193aebb7b9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731883252747 2024-11-17T22:40:53,838 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51b8604e80e8479181b83fb6cfdbe56a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731883253108 2024-11-17T22:40:53,838 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d25964c98cc747b8bff060f579a5c592, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731883253108 2024-11-17T22:40:53,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:53,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/7ad6caf683914140a1c2c7bb8d2d7f95 is 50, key is test_row_0/A:col10/1731883253281/Put/seqid=0 2024-11-17T22:40:53,859 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#A#compaction#210 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:53,860 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ad7fa1f9dfb34519b11ef83cc40d8223 is 50, key is test_row_0/A:col10/1731883253113/Put/seqid=0 2024-11-17T22:40:53,872 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#B#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:53,873 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/e022dbc759c04fb8b171d5b1bfe21e1d is 50, key is test_row_0/B:col10/1731883253113/Put/seqid=0 2024-11-17T22:40:53,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742077_1253 (size=12409) 2024-11-17T22:40:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:53,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:53,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742076_1252 (size=12151) 2024-11-17T22:40:53,927 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/7ad6caf683914140a1c2c7bb8d2d7f95 2024-11-17T22:40:53,930 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ad7fa1f9dfb34519b11ef83cc40d8223 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ad7fa1f9dfb34519b11ef83cc40d8223 2024-11-17T22:40:53,945 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/A of ad1461c99ce1acccd0c930b472c9a7c1 into ad7fa1f9dfb34519b11ef83cc40d8223(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:53,945 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:53,945 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/A, priority=13, startTime=1731883253834; duration=0sec 2024-11-17T22:40:53,945 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:53,945 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:A 2024-11-17T22:40:53,945 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:53,947 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:53,947 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/C is initiating minor compaction (all files) 2024-11-17T22:40:53,947 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/C in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:53,947 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/ce7ea50943214167a3508be4efc159d2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/fa1b1445d2c544ccb2bd6129d68c2c52, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0586271406c942c7b5912ceefb3c3741] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=35.5 K 2024-11-17T22:40:53,948 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce7ea50943214167a3508be4efc159d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731883252587 2024-11-17T22:40:53,948 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa1b1445d2c544ccb2bd6129d68c2c52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731883252747 2024-11-17T22:40:53,949 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0586271406c942c7b5912ceefb3c3741, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731883253108 2024-11-17T22:40:53,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742078_1254 (size=12409) 2024-11-17T22:40:53,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/02ca8c1fd3f74f1baa92e734eb5f1c0e is 50, key is test_row_0/B:col10/1731883253281/Put/seqid=0 2024-11-17T22:40:53,962 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/e022dbc759c04fb8b171d5b1bfe21e1d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e022dbc759c04fb8b171d5b1bfe21e1d 2024-11-17T22:40:53,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883313957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883313958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883313961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,967 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#C#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:53,968 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/7598b3b647d7440788d9db93e095c4f6 is 50, key is test_row_0/C:col10/1731883253113/Put/seqid=0 2024-11-17T22:40:53,970 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/B of ad1461c99ce1acccd0c930b472c9a7c1 into e022dbc759c04fb8b171d5b1bfe21e1d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:53,970 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:53,970 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/B, priority=13, startTime=1731883253834; duration=0sec 2024-11-17T22:40:53,970 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:53,970 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:B 2024-11-17T22:40:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883313965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:53,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742079_1255 (size=12151) 2024-11-17T22:40:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T22:40:54,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742080_1256 (size=12409) 2024-11-17T22:40:54,042 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/7598b3b647d7440788d9db93e095c4f6 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7598b3b647d7440788d9db93e095c4f6 2024-11-17T22:40:54,051 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/C of ad1461c99ce1acccd0c930b472c9a7c1 into 7598b3b647d7440788d9db93e095c4f6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:54,051 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:54,051 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/C, priority=13, startTime=1731883253849; duration=0sec 2024-11-17T22:40:54,051 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:54,052 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:C 2024-11-17T22:40:54,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883314067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883314077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883314077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883314077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883314277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883314287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883314287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883314288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T22:40:54,392 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/02ca8c1fd3f74f1baa92e734eb5f1c0e 2024-11-17T22:40:54,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/1757848d5bc64827a44dd2237d1fb97b is 50, key is test_row_0/C:col10/1731883253281/Put/seqid=0 2024-11-17T22:40:54,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742081_1257 (size=12151) 2024-11-17T22:40:54,467 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/1757848d5bc64827a44dd2237d1fb97b 2024-11-17T22:40:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/7ad6caf683914140a1c2c7bb8d2d7f95 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/7ad6caf683914140a1c2c7bb8d2d7f95 2024-11-17T22:40:54,478 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/7ad6caf683914140a1c2c7bb8d2d7f95, entries=150, sequenceid=154, filesize=11.9 K 2024-11-17T22:40:54,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/02ca8c1fd3f74f1baa92e734eb5f1c0e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/02ca8c1fd3f74f1baa92e734eb5f1c0e 2024-11-17T22:40:54,486 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/02ca8c1fd3f74f1baa92e734eb5f1c0e, entries=150, sequenceid=154, filesize=11.9 K 2024-11-17T22:40:54,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/1757848d5bc64827a44dd2237d1fb97b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/1757848d5bc64827a44dd2237d1fb97b 2024-11-17T22:40:54,494 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/1757848d5bc64827a44dd2237d1fb97b, entries=150, sequenceid=154, filesize=11.9 K 2024-11-17T22:40:54,495 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for ad1461c99ce1acccd0c930b472c9a7c1 in 658ms, sequenceid=154, compaction requested=false 2024-11-17T22:40:54,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:54,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:54,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-17T22:40:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-17T22:40:54,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-17T22:40:54,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 815 msec 2024-11-17T22:40:54,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 821 msec 2024-11-17T22:40:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:54,590 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:54,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T22:40:54,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/18bab74f5331464681dd2a943af1cd4e is 50, key is test_row_0/A:col10/1731883254589/Put/seqid=0 2024-11-17T22:40:54,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742082_1258 (size=14541) 2024-11-17T22:40:54,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883314619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883314621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883314622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883314622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883314623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883314727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883314737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883314737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883314737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883314737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T22:40:54,817 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:54,817 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-17T22:40:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T22:40:54,819 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:54,820 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:54,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T22:40:54,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883314937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883314947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883314947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883314948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883314948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:54,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T22:40:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:54,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:55,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/18bab74f5331464681dd2a943af1cd4e 2024-11-17T22:40:55,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/3b7c11ecebba4ae7920b173707918b57 is 50, key is test_row_0/B:col10/1731883254589/Put/seqid=0 2024-11-17T22:40:55,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742083_1259 (size=12151) 2024-11-17T22:40:55,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/3b7c11ecebba4ae7920b173707918b57 2024-11-17T22:40:55,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4ad0ee705d804f1588fd4370f1d68164 is 50, key is test_row_0/C:col10/1731883254589/Put/seqid=0 2024-11-17T22:40:55,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742084_1260 (size=12151) 2024-11-17T22:40:55,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4ad0ee705d804f1588fd4370f1d68164 2024-11-17T22:40:55,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/18bab74f5331464681dd2a943af1cd4e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/18bab74f5331464681dd2a943af1cd4e 2024-11-17T22:40:55,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/18bab74f5331464681dd2a943af1cd4e, entries=200, sequenceid=175, filesize=14.2 K 2024-11-17T22:40:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/3b7c11ecebba4ae7920b173707918b57 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/3b7c11ecebba4ae7920b173707918b57 2024-11-17T22:40:55,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/3b7c11ecebba4ae7920b173707918b57, entries=150, sequenceid=175, filesize=11.9 K 2024-11-17T22:40:55,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4ad0ee705d804f1588fd4370f1d68164 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4ad0ee705d804f1588fd4370f1d68164 2024-11-17T22:40:55,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4ad0ee705d804f1588fd4370f1d68164, entries=150, sequenceid=175, filesize=11.9 K 2024-11-17T22:40:55,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for ad1461c99ce1acccd0c930b472c9a7c1 in 487ms, sequenceid=175, compaction requested=true 2024-11-17T22:40:55,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:55,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:55,077 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:55,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:55,077 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:55,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:55,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:55,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:55,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:55,079 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:55,079 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/A is initiating minor compaction (all files) 2024-11-17T22:40:55,079 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/A in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:55,079 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ad7fa1f9dfb34519b11ef83cc40d8223, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/7ad6caf683914140a1c2c7bb8d2d7f95, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/18bab74f5331464681dd2a943af1cd4e] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=38.2 K 2024-11-17T22:40:55,080 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad7fa1f9dfb34519b11ef83cc40d8223, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731883253108 2024-11-17T22:40:55,081 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ad6caf683914140a1c2c7bb8d2d7f95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731883253281 2024-11-17T22:40:55,081 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18bab74f5331464681dd2a943af1cd4e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731883253939 2024-11-17T22:40:55,084 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:55,084 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/B is initiating minor compaction (all files) 2024-11-17T22:40:55,085 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/B in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:55,085 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e022dbc759c04fb8b171d5b1bfe21e1d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/02ca8c1fd3f74f1baa92e734eb5f1c0e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/3b7c11ecebba4ae7920b173707918b57] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=35.9 K 2024-11-17T22:40:55,086 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting e022dbc759c04fb8b171d5b1bfe21e1d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731883253108 2024-11-17T22:40:55,086 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ca8c1fd3f74f1baa92e734eb5f1c0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731883253281 2024-11-17T22:40:55,087 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b7c11ecebba4ae7920b173707918b57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731883253939 2024-11-17T22:40:55,109 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#A#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:55,110 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/d4db0d4b2dc145d89525a4772c0ada07 is 50, key is test_row_0/A:col10/1731883254589/Put/seqid=0 2024-11-17T22:40:55,123 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:55,124 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/483b6060fa4e417789874fa7b08a827b is 50, key is test_row_0/B:col10/1731883254589/Put/seqid=0 2024-11-17T22:40:55,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T22:40:55,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:55,127 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T22:40:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T22:40:55,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ee5e197e255643f6b690f79cb0e0fb2b is 50, key is test_row_0/A:col10/1731883254618/Put/seqid=0 2024-11-17T22:40:55,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742085_1261 (size=12561) 2024-11-17T22:40:55,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742086_1262 (size=12561) 2024-11-17T22:40:55,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742087_1263 (size=12151) 2024-11-17T22:40:55,229 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/483b6060fa4e417789874fa7b08a827b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/483b6060fa4e417789874fa7b08a827b 2024-11-17T22:40:55,236 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/B of ad1461c99ce1acccd0c930b472c9a7c1 into 483b6060fa4e417789874fa7b08a827b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:55,236 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:55,236 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/B, priority=13, startTime=1731883255077; duration=0sec 2024-11-17T22:40:55,236 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:55,236 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:B 2024-11-17T22:40:55,236 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:55,238 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:55,238 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/C is initiating minor compaction (all files) 2024-11-17T22:40:55,238 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/C in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:55,238 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7598b3b647d7440788d9db93e095c4f6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/1757848d5bc64827a44dd2237d1fb97b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4ad0ee705d804f1588fd4370f1d68164] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=35.9 K 2024-11-17T22:40:55,239 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 7598b3b647d7440788d9db93e095c4f6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731883253108 2024-11-17T22:40:55,239 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 1757848d5bc64827a44dd2237d1fb97b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731883253281 2024-11-17T22:40:55,239 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ad0ee705d804f1588fd4370f1d68164, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731883253939 2024-11-17T22:40:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:55,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:55,252 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#C#compaction#221 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:55,252 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/0b7c7b67e9bf4b28b72c4651ae530e25 is 50, key is test_row_0/C:col10/1731883254589/Put/seqid=0 2024-11-17T22:40:55,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883315272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883315273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883315273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883315276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883315278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742088_1264 (size=12561) 2024-11-17T22:40:55,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883315387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883315387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883315387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883315387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883315387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T22:40:55,598 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/d4db0d4b2dc145d89525a4772c0ada07 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d4db0d4b2dc145d89525a4772c0ada07 2024-11-17T22:40:55,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883315600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883315600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883315605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883315605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883315605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,613 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/A of ad1461c99ce1acccd0c930b472c9a7c1 into d4db0d4b2dc145d89525a4772c0ada07(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:55,613 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:55,613 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/A, priority=13, startTime=1731883255077; duration=0sec 2024-11-17T22:40:55,613 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:55,613 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:A 2024-11-17T22:40:55,621 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ee5e197e255643f6b690f79cb0e0fb2b 2024-11-17T22:40:55,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b287de9a44be491492301bbfe0b55662 is 50, key is test_row_0/B:col10/1731883254618/Put/seqid=0 2024-11-17T22:40:55,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742089_1265 (size=12151) 2024-11-17T22:40:55,697 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/0b7c7b67e9bf4b28b72c4651ae530e25 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0b7c7b67e9bf4b28b72c4651ae530e25 2024-11-17T22:40:55,705 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/C of ad1461c99ce1acccd0c930b472c9a7c1 into 0b7c7b67e9bf4b28b72c4651ae530e25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:55,705 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:55,705 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/C, priority=13, startTime=1731883255078; duration=0sec 2024-11-17T22:40:55,705 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:55,705 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:C 2024-11-17T22:40:55,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883315908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883315908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883315917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883315917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883315918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T22:40:56,080 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b287de9a44be491492301bbfe0b55662 2024-11-17T22:40:56,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/ebbb9f0cd573434fa49c75059b1a7922 is 50, key is test_row_0/C:col10/1731883254618/Put/seqid=0 2024-11-17T22:40:56,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742090_1266 (size=12151) 2024-11-17T22:40:56,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883316417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:56,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883316417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:56,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883316427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883316427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883316427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:56,507 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/ebbb9f0cd573434fa49c75059b1a7922 2024-11-17T22:40:56,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ee5e197e255643f6b690f79cb0e0fb2b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ee5e197e255643f6b690f79cb0e0fb2b 2024-11-17T22:40:56,521 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ee5e197e255643f6b690f79cb0e0fb2b, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T22:40:56,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b287de9a44be491492301bbfe0b55662 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b287de9a44be491492301bbfe0b55662 2024-11-17T22:40:56,528 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b287de9a44be491492301bbfe0b55662, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T22:40:56,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/ebbb9f0cd573434fa49c75059b1a7922 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/ebbb9f0cd573434fa49c75059b1a7922 2024-11-17T22:40:56,535 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/ebbb9f0cd573434fa49c75059b1a7922, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T22:40:56,537 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for ad1461c99ce1acccd0c930b472c9a7c1 in 1411ms, sequenceid=193, compaction requested=false 2024-11-17T22:40:56,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:56,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:56,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-17T22:40:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-17T22:40:56,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-17T22:40:56,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7190 sec 2024-11-17T22:40:56,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.7290 sec 2024-11-17T22:40:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T22:40:56,968 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:56,968 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-17T22:40:56,970 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T22:40:56,971 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:56,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T22:40:57,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T22:40:57,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:57,128 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T22:40:57,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ddafb3ba44954287a3b08f8b43e332d7 is 50, key is test_row_0/A:col10/1731883255275/Put/seqid=0 2024-11-17T22:40:57,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742091_1267 (size=9757) 2024-11-17T22:40:57,158 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ddafb3ba44954287a3b08f8b43e332d7 2024-11-17T22:40:57,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b38b9fa652a7429ab3995c1348be438c is 50, key is test_row_0/B:col10/1731883255275/Put/seqid=0 2024-11-17T22:40:57,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742092_1268 (size=9757) 2024-11-17T22:40:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T22:40:57,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:57,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883317488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883317495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883317495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883317495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883317495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T22:40:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883317597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883317606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883317606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883317606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883317606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,619 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b38b9fa652a7429ab3995c1348be438c 2024-11-17T22:40:57,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/2d62f133ebfb4a1d99bb20e14e807dba is 50, key is test_row_0/C:col10/1731883255275/Put/seqid=0 2024-11-17T22:40:57,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742093_1269 (size=9757) 2024-11-17T22:40:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883317807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883317816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883317816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883317817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883317817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,029 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/2d62f133ebfb4a1d99bb20e14e807dba 2024-11-17T22:40:58,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/ddafb3ba44954287a3b08f8b43e332d7 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ddafb3ba44954287a3b08f8b43e332d7 2024-11-17T22:40:58,037 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ddafb3ba44954287a3b08f8b43e332d7, entries=100, sequenceid=214, filesize=9.5 K 2024-11-17T22:40:58,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b38b9fa652a7429ab3995c1348be438c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b38b9fa652a7429ab3995c1348be438c 2024-11-17T22:40:58,044 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b38b9fa652a7429ab3995c1348be438c, entries=100, sequenceid=214, filesize=9.5 K 2024-11-17T22:40:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/2d62f133ebfb4a1d99bb20e14e807dba as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2d62f133ebfb4a1d99bb20e14e807dba 2024-11-17T22:40:58,049 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2d62f133ebfb4a1d99bb20e14e807dba, entries=100, sequenceid=214, filesize=9.5 K 2024-11-17T22:40:58,050 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for ad1461c99ce1acccd0c930b472c9a7c1 in 922ms, sequenceid=214, compaction requested=true 2024-11-17T22:40:58,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:58,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:58,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-17T22:40:58,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-17T22:40:58,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-17T22:40:58,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0800 sec 2024-11-17T22:40:58,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0850 sec 2024-11-17T22:40:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T22:40:58,107 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:58,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-17T22:40:58,110 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T22:40:58,110 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:58,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:58,122 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:58,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-17T22:40:58,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/23721b5265fa4ab7b2b3392d17c24b83 is 50, key is test_row_0/A:col10/1731883257484/Put/seqid=0 2024-11-17T22:40:58,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883318142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883318144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883318144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883318145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883318147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742094_1270 (size=14541) 2024-11-17T22:40:58,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/23721b5265fa4ab7b2b3392d17c24b83 2024-11-17T22:40:58,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/021bb82f0ca14e428af278a849e9c804 is 50, key is test_row_0/B:col10/1731883257484/Put/seqid=0 2024-11-17T22:40:58,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742095_1271 (size=12151) 2024-11-17T22:40:58,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/021bb82f0ca14e428af278a849e9c804 2024-11-17T22:40:58,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/71517a39671944af8bcf11a3a569d48d is 50, key is test_row_0/C:col10/1731883257484/Put/seqid=0 2024-11-17T22:40:58,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742096_1272 (size=12151) 2024-11-17T22:40:58,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/71517a39671944af8bcf11a3a569d48d 2024-11-17T22:40:58,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/23721b5265fa4ab7b2b3392d17c24b83 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/23721b5265fa4ab7b2b3392d17c24b83 2024-11-17T22:40:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T22:40:58,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/23721b5265fa4ab7b2b3392d17c24b83, entries=200, sequenceid=233, filesize=14.2 K 2024-11-17T22:40:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/021bb82f0ca14e428af278a849e9c804 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/021bb82f0ca14e428af278a849e9c804 2024-11-17T22:40:58,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/021bb82f0ca14e428af278a849e9c804, entries=150, sequenceid=233, filesize=11.9 K 2024-11-17T22:40:58,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/71517a39671944af8bcf11a3a569d48d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/71517a39671944af8bcf11a3a569d48d 2024-11-17T22:40:58,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/71517a39671944af8bcf11a3a569d48d, entries=150, sequenceid=233, filesize=11.9 K 2024-11-17T22:40:58,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for ad1461c99ce1acccd0c930b472c9a7c1 in 114ms, sequenceid=233, compaction requested=true 2024-11-17T22:40:58,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:58,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:58,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:58,237 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:58,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:58,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:58,237 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:58,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:58,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:58,238 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:58,238 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/B is initiating minor compaction (all files) 2024-11-17T22:40:58,239 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/B in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:58,239 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/483b6060fa4e417789874fa7b08a827b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b287de9a44be491492301bbfe0b55662, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b38b9fa652a7429ab3995c1348be438c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/021bb82f0ca14e428af278a849e9c804] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=45.5 K 2024-11-17T22:40:58,239 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49010 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:58,239 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/A is initiating minor compaction (all files) 2024-11-17T22:40:58,239 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/A in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:58,239 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d4db0d4b2dc145d89525a4772c0ada07, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ee5e197e255643f6b690f79cb0e0fb2b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ddafb3ba44954287a3b08f8b43e332d7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/23721b5265fa4ab7b2b3392d17c24b83] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=47.9 K 2024-11-17T22:40:58,241 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 483b6060fa4e417789874fa7b08a827b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731883253939 2024-11-17T22:40:58,241 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4db0d4b2dc145d89525a4772c0ada07, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731883253939 2024-11-17T22:40:58,241 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b287de9a44be491492301bbfe0b55662, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731883254616 2024-11-17T22:40:58,241 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee5e197e255643f6b690f79cb0e0fb2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731883254616 2024-11-17T22:40:58,242 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b38b9fa652a7429ab3995c1348be438c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731883255272 2024-11-17T22:40:58,242 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddafb3ba44954287a3b08f8b43e332d7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731883255272 2024-11-17T22:40:58,242 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 021bb82f0ca14e428af278a849e9c804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731883257484 2024-11-17T22:40:58,242 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23721b5265fa4ab7b2b3392d17c24b83, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731883257484 2024-11-17T22:40:58,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T22:40:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:58,264 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-17T22:40:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:58,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:58,268 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#A#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:58,268 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/6323d7ed618b4947954ae5f01b896d29 is 50, key is test_row_0/A:col10/1731883257484/Put/seqid=0 2024-11-17T22:40:58,268 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:58,269 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b20a894fd1284115a9590d3fe31d7f12 is 50, key is test_row_0/B:col10/1731883257484/Put/seqid=0 2024-11-17T22:40:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/29bf0173d85a49bab9b4f3b8d133bbbc is 50, key is test_row_0/A:col10/1731883258142/Put/seqid=0 2024-11-17T22:40:58,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883318292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883318294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883318301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883318306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883318307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742098_1274 (size=12697) 2024-11-17T22:40:58,326 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b20a894fd1284115a9590d3fe31d7f12 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b20a894fd1284115a9590d3fe31d7f12 2024-11-17T22:40:58,333 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/B of ad1461c99ce1acccd0c930b472c9a7c1 into b20a894fd1284115a9590d3fe31d7f12(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:58,333 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:58,333 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/B, priority=12, startTime=1731883258237; duration=0sec 2024-11-17T22:40:58,333 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:58,333 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:B 2024-11-17T22:40:58,333 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:40:58,335 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:40:58,335 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/C is initiating minor compaction (all files) 2024-11-17T22:40:58,335 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/C in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:58,335 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0b7c7b67e9bf4b28b72c4651ae530e25, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/ebbb9f0cd573434fa49c75059b1a7922, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2d62f133ebfb4a1d99bb20e14e807dba, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/71517a39671944af8bcf11a3a569d48d] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=45.5 K 2024-11-17T22:40:58,336 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b7c7b67e9bf4b28b72c4651ae530e25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731883253939 2024-11-17T22:40:58,337 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting ebbb9f0cd573434fa49c75059b1a7922, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731883254616 2024-11-17T22:40:58,338 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d62f133ebfb4a1d99bb20e14e807dba, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731883255272 2024-11-17T22:40:58,338 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 71517a39671944af8bcf11a3a569d48d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731883257484 2024-11-17T22:40:58,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742099_1275 (size=12151) 2024-11-17T22:40:58,347 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/29bf0173d85a49bab9b4f3b8d133bbbc 2024-11-17T22:40:58,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/2c7e6c49d19a4bf88ffb8b40ae4d664e is 50, key is test_row_0/B:col10/1731883258142/Put/seqid=0 2024-11-17T22:40:58,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742097_1273 (size=12697) 2024-11-17T22:40:58,372 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:58,373 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4794f5b3317141d7bdc8db0e07fde8b8 is 50, key is test_row_0/C:col10/1731883257484/Put/seqid=0 2024-11-17T22:40:58,383 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/6323d7ed618b4947954ae5f01b896d29 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/6323d7ed618b4947954ae5f01b896d29 2024-11-17T22:40:58,389 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/A of ad1461c99ce1acccd0c930b472c9a7c1 into 6323d7ed618b4947954ae5f01b896d29(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:58,389 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:58,390 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/A, priority=12, startTime=1731883258237; duration=0sec 2024-11-17T22:40:58,390 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:58,390 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:A 2024-11-17T22:40:58,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742101_1277 (size=12697) 2024-11-17T22:40:58,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742100_1276 (size=12151) 2024-11-17T22:40:58,401 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/2c7e6c49d19a4bf88ffb8b40ae4d664e 2024-11-17T22:40:58,402 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4794f5b3317141d7bdc8db0e07fde8b8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4794f5b3317141d7bdc8db0e07fde8b8 2024-11-17T22:40:58,410 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/C of ad1461c99ce1acccd0c930b472c9a7c1 into 4794f5b3317141d7bdc8db0e07fde8b8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:58,410 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:58,410 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/C, priority=12, startTime=1731883258237; duration=0sec 2024-11-17T22:40:58,410 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:58,410 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:C 2024-11-17T22:40:58,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4f3f210a347e490dbab0953554d06021 is 50, key is test_row_0/C:col10/1731883258142/Put/seqid=0 2024-11-17T22:40:58,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883318417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883318417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883318417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883318417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T22:40:58,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883318429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742102_1278 (size=12151) 2024-11-17T22:40:58,468 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4f3f210a347e490dbab0953554d06021 2024-11-17T22:40:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/29bf0173d85a49bab9b4f3b8d133bbbc as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/29bf0173d85a49bab9b4f3b8d133bbbc 2024-11-17T22:40:58,479 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/29bf0173d85a49bab9b4f3b8d133bbbc, entries=150, sequenceid=251, filesize=11.9 K 2024-11-17T22:40:58,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/2c7e6c49d19a4bf88ffb8b40ae4d664e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2c7e6c49d19a4bf88ffb8b40ae4d664e 2024-11-17T22:40:58,485 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2c7e6c49d19a4bf88ffb8b40ae4d664e, entries=150, sequenceid=251, filesize=11.9 K 2024-11-17T22:40:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4f3f210a347e490dbab0953554d06021 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f3f210a347e490dbab0953554d06021 2024-11-17T22:40:58,493 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f3f210a347e490dbab0953554d06021, entries=150, sequenceid=251, filesize=11.9 K 2024-11-17T22:40:58,494 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for ad1461c99ce1acccd0c930b472c9a7c1 in 231ms, sequenceid=251, compaction requested=false 2024-11-17T22:40:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-17T22:40:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-17T22:40:58,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-17T22:40:58,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 387 msec 2024-11-17T22:40:58,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 393 msec 2024-11-17T22:40:58,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T22:40:58,618 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T22:40:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:58,629 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:40:58,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T22:40:58,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/4fa56db47ee249adb14e91004d886d5e is 50, key is test_row_0/A:col10/1731883258290/Put/seqid=0 2024-11-17T22:40:58,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742103_1279 (size=12301) 2024-11-17T22:40:58,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883318645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883318647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883318647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883318651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883318652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T22:40:58,737 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:40:58,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:40:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-17T22:40:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T22:40:58,739 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:40:58,739 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:40:58,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:40:58,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883318757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883318757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883318767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883318767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883318767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T22:40:58,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T22:40:58,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:58,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:58,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:58,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:58,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:58,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883318967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883318967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883318977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883318977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:58,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883318977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:58,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/4fa56db47ee249adb14e91004d886d5e 2024-11-17T22:40:59,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T22:40:59,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:59,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/553adb3aac7a43c499ee38620c72219d is 50, key is test_row_0/B:col10/1731883258290/Put/seqid=0 2024-11-17T22:40:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T22:40:59,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742104_1280 (size=12301) 2024-11-17T22:40:59,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T22:40:59,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:59,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883319277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883319277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883319287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883319287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883319287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T22:40:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T22:40:59,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/553adb3aac7a43c499ee38620c72219d 2024-11-17T22:40:59,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/53c556d7b70e490c9f1ec51be6b3e6d9 is 50, key is test_row_0/C:col10/1731883258290/Put/seqid=0 2024-11-17T22:40:59,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T22:40:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:40:59,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742105_1281 (size=12301) 2024-11-17T22:40:59,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/53c556d7b70e490c9f1ec51be6b3e6d9 2024-11-17T22:40:59,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/4fa56db47ee249adb14e91004d886d5e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/4fa56db47ee249adb14e91004d886d5e 2024-11-17T22:40:59,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/4fa56db47ee249adb14e91004d886d5e, entries=150, sequenceid=275, filesize=12.0 K 2024-11-17T22:40:59,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/553adb3aac7a43c499ee38620c72219d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/553adb3aac7a43c499ee38620c72219d 2024-11-17T22:40:59,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/553adb3aac7a43c499ee38620c72219d, entries=150, sequenceid=275, filesize=12.0 K 2024-11-17T22:40:59,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/53c556d7b70e490c9f1ec51be6b3e6d9 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/53c556d7b70e490c9f1ec51be6b3e6d9 2024-11-17T22:40:59,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/53c556d7b70e490c9f1ec51be6b3e6d9, entries=150, sequenceid=275, filesize=12.0 K 2024-11-17T22:40:59,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for ad1461c99ce1acccd0c930b472c9a7c1 in 934ms, sequenceid=275, compaction requested=true 2024-11-17T22:40:59,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:40:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:59,564 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:40:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:59,564 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:40:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:59,565 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:59,565 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:59,565 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/B is initiating minor compaction (all files) 2024-11-17T22:40:59,565 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/A is initiating minor compaction (all files) 2024-11-17T22:40:59,565 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/A in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,565 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/B in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,565 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b20a894fd1284115a9590d3fe31d7f12, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2c7e6c49d19a4bf88ffb8b40ae4d664e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/553adb3aac7a43c499ee38620c72219d] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=36.3 K 2024-11-17T22:40:59,565 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/6323d7ed618b4947954ae5f01b896d29, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/29bf0173d85a49bab9b4f3b8d133bbbc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/4fa56db47ee249adb14e91004d886d5e] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=36.3 K 2024-11-17T22:40:59,565 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b20a894fd1284115a9590d3fe31d7f12, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731883257484 2024-11-17T22:40:59,565 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6323d7ed618b4947954ae5f01b896d29, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731883257484 2024-11-17T22:40:59,565 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29bf0173d85a49bab9b4f3b8d133bbbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731883258139 2024-11-17T22:40:59,566 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c7e6c49d19a4bf88ffb8b40ae4d664e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731883258139 2024-11-17T22:40:59,566 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fa56db47ee249adb14e91004d886d5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731883258290 2024-11-17T22:40:59,566 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 553adb3aac7a43c499ee38620c72219d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731883258290 2024-11-17T22:40:59,575 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#B#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:59,575 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/20cd98b371784a1ea90b564b3c0ec173 is 50, key is test_row_0/B:col10/1731883258290/Put/seqid=0 2024-11-17T22:40:59,579 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#A#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:59,580 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/dc83d7008f1649d3bbea1d14b719cd3b is 50, key is test_row_0/A:col10/1731883258290/Put/seqid=0 2024-11-17T22:40:59,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742106_1282 (size=12949) 2024-11-17T22:40:59,617 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/20cd98b371784a1ea90b564b3c0ec173 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/20cd98b371784a1ea90b564b3c0ec173 2024-11-17T22:40:59,624 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/B of ad1461c99ce1acccd0c930b472c9a7c1 into 20cd98b371784a1ea90b564b3c0ec173(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:59,624 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:59,625 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/B, priority=13, startTime=1731883259564; duration=0sec 2024-11-17T22:40:59,625 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:40:59,625 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:B 2024-11-17T22:40:59,625 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:40:59,626 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:40:59,627 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/C is initiating minor compaction (all files) 2024-11-17T22:40:59,627 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/C in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,627 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4794f5b3317141d7bdc8db0e07fde8b8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f3f210a347e490dbab0953554d06021, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/53c556d7b70e490c9f1ec51be6b3e6d9] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=36.3 K 2024-11-17T22:40:59,627 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4794f5b3317141d7bdc8db0e07fde8b8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731883257484 2024-11-17T22:40:59,627 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3f210a347e490dbab0953554d06021, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731883258139 2024-11-17T22:40:59,628 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 53c556d7b70e490c9f1ec51be6b3e6d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731883258290 2024-11-17T22:40:59,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742107_1283 (size=12949) 2024-11-17T22:40:59,643 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/dc83d7008f1649d3bbea1d14b719cd3b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/dc83d7008f1649d3bbea1d14b719cd3b 2024-11-17T22:40:59,646 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#C#compaction#241 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:40:59,649 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4bc87ef0be144160a1d4c23cfc1fe3f4 is 50, key is test_row_0/C:col10/1731883258290/Put/seqid=0 2024-11-17T22:40:59,655 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/A of ad1461c99ce1acccd0c930b472c9a7c1 into dc83d7008f1649d3bbea1d14b719cd3b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:40:59,655 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:40:59,656 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/A, priority=13, startTime=1731883259563; duration=0sec 2024-11-17T22:40:59,657 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:40:59,657 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:A 2024-11-17T22:40:59,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T22:40:59,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:40:59,669 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T22:40:59,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/debc46a123d543c99f0db99e63b151d7 is 50, key is test_row_0/A:col10/1731883258645/Put/seqid=0 2024-11-17T22:40:59,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742108_1284 (size=12949) 2024-11-17T22:40:59,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742109_1285 (size=12301) 2024-11-17T22:40:59,720 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/debc46a123d543c99f0db99e63b151d7 2024-11-17T22:40:59,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/805782a0bfc44fc88a080d4edab8033e is 50, key is test_row_0/B:col10/1731883258645/Put/seqid=0 2024-11-17T22:40:59,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742110_1286 (size=12301) 2024-11-17T22:40:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:40:59,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:40:59,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883319807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883319809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883319811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883319812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883319812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T22:40:59,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883319917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883319918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883319926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883319926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:40:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883319927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:40:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:40:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:40:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,105 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4bc87ef0be144160a1d4c23cfc1fe3f4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4bc87ef0be144160a1d4c23cfc1fe3f4 2024-11-17T22:41:00,112 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/C of ad1461c99ce1acccd0c930b472c9a7c1 into 4bc87ef0be144160a1d4c23cfc1fe3f4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:00,113 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:00,113 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/C, priority=13, startTime=1731883259564; duration=0sec 2024-11-17T22:41:00,113 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:00,113 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:C 2024-11-17T22:41:00,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883320127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883320127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883320137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883320137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883320137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,165 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/805782a0bfc44fc88a080d4edab8033e 2024-11-17T22:41:00,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/a3d49aea18cb4598b1b43da62a84acaf is 50, key is test_row_0/C:col10/1731883258645/Put/seqid=0 2024-11-17T22:41:00,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742111_1287 (size=12301) 2024-11-17T22:41:00,208 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/a3d49aea18cb4598b1b43da62a84acaf 2024-11-17T22:41:00,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/debc46a123d543c99f0db99e63b151d7 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/debc46a123d543c99f0db99e63b151d7 2024-11-17T22:41:00,223 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/debc46a123d543c99f0db99e63b151d7, entries=150, sequenceid=293, filesize=12.0 K 2024-11-17T22:41:00,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/805782a0bfc44fc88a080d4edab8033e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/805782a0bfc44fc88a080d4edab8033e 2024-11-17T22:41:00,232 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/805782a0bfc44fc88a080d4edab8033e, entries=150, sequenceid=293, filesize=12.0 K 2024-11-17T22:41:00,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/a3d49aea18cb4598b1b43da62a84acaf as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/a3d49aea18cb4598b1b43da62a84acaf 2024-11-17T22:41:00,239 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/a3d49aea18cb4598b1b43da62a84acaf, entries=150, sequenceid=293, filesize=12.0 K 2024-11-17T22:41:00,240 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for ad1461c99ce1acccd0c930b472c9a7c1 in 571ms, sequenceid=293, compaction requested=false 2024-11-17T22:41:00,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:00,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:00,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-17T22:41:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-17T22:41:00,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-17T22:41:00,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5070 sec 2024-11-17T22:41:00,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.5140 sec 2024-11-17T22:41:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:00,439 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:00,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T22:41:00,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/d57409bd30f54b959f0a71ad8c2c4731 is 50, key is test_row_0/A:col10/1731883259806/Put/seqid=0 2024-11-17T22:41:00,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742112_1288 (size=12301) 2024-11-17T22:41:00,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/d57409bd30f54b959f0a71ad8c2c4731 2024-11-17T22:41:00,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/9c6748a662fe4910a9eb46cb2f20e5aa is 50, key is test_row_0/B:col10/1731883259806/Put/seqid=0 2024-11-17T22:41:00,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742113_1289 (size=12301) 2024-11-17T22:41:00,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883320456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883320458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883320460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883320461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883320462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883320567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883320567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883320577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883320577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883320577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883320777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883320777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883320787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883320787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883320787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:00,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/9c6748a662fe4910a9eb46cb2f20e5aa 2024-11-17T22:41:00,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/38ab749d024f41ffb8897d14c2219f91 is 50, key is test_row_0/C:col10/1731883259806/Put/seqid=0 2024-11-17T22:41:00,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742114_1290 (size=12301) 2024-11-17T22:41:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T22:41:00,887 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:00,888 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-17T22:41:00,889 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T22:41:00,890 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:00,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T22:41:01,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-17T22:41:01,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:01,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:01,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:01,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:01,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:01,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883321087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883321087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883321097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883321097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883321097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-17T22:41:01,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:01,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:01,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:01,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:01,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T22:41:01,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/38ab749d024f41ffb8897d14c2219f91 2024-11-17T22:41:01,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/d57409bd30f54b959f0a71ad8c2c4731 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d57409bd30f54b959f0a71ad8c2c4731 2024-11-17T22:41:01,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d57409bd30f54b959f0a71ad8c2c4731, entries=150, sequenceid=315, filesize=12.0 K 2024-11-17T22:41:01,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/9c6748a662fe4910a9eb46cb2f20e5aa as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/9c6748a662fe4910a9eb46cb2f20e5aa 2024-11-17T22:41:01,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/9c6748a662fe4910a9eb46cb2f20e5aa, entries=150, sequenceid=315, filesize=12.0 K 2024-11-17T22:41:01,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/38ab749d024f41ffb8897d14c2219f91 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/38ab749d024f41ffb8897d14c2219f91 2024-11-17T22:41:01,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/38ab749d024f41ffb8897d14c2219f91, entries=150, sequenceid=315, filesize=12.0 K 2024-11-17T22:41:01,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for ad1461c99ce1acccd0c930b472c9a7c1 in 860ms, sequenceid=315, compaction requested=true 2024-11-17T22:41:01,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:01,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:01,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:01,300 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:01,300 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:01,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:01,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:01,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:01,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:01,302 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:01,302 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/B is initiating minor compaction (all files) 2024-11-17T22:41:01,302 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/B in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:01,302 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/20cd98b371784a1ea90b564b3c0ec173, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/805782a0bfc44fc88a080d4edab8033e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/9c6748a662fe4910a9eb46cb2f20e5aa] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=36.7 K 2024-11-17T22:41:01,303 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:01,303 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/A is initiating minor compaction (all files) 2024-11-17T22:41:01,303 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/A in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:01,303 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/dc83d7008f1649d3bbea1d14b719cd3b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/debc46a123d543c99f0db99e63b151d7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d57409bd30f54b959f0a71ad8c2c4731] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=36.7 K 2024-11-17T22:41:01,303 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 20cd98b371784a1ea90b564b3c0ec173, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731883258290 2024-11-17T22:41:01,303 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc83d7008f1649d3bbea1d14b719cd3b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731883258290 2024-11-17T22:41:01,304 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 805782a0bfc44fc88a080d4edab8033e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731883258645 2024-11-17T22:41:01,304 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c6748a662fe4910a9eb46cb2f20e5aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731883259806 2024-11-17T22:41:01,304 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting debc46a123d543c99f0db99e63b151d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731883258645 2024-11-17T22:41:01,305 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting d57409bd30f54b959f0a71ad8c2c4731, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731883259806 2024-11-17T22:41:01,319 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#B#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:01,320 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/92dcb3fcd66f48a6b100248db8f1dba1 is 50, key is test_row_0/B:col10/1731883259806/Put/seqid=0 2024-11-17T22:41:01,322 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#A#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:01,329 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/f18062bdf035483d8362ca80fae4f81c is 50, key is test_row_0/A:col10/1731883259806/Put/seqid=0 2024-11-17T22:41:01,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742115_1291 (size=13051) 2024-11-17T22:41:01,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-17T22:41:01,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:01,350 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T22:41:01,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/03662fa777b34685bedfccd9aa45c76f is 50, key is test_row_0/A:col10/1731883260451/Put/seqid=0 2024-11-17T22:41:01,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742116_1292 (size=13051) 2024-11-17T22:41:01,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742117_1293 (size=12301) 2024-11-17T22:41:01,363 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/03662fa777b34685bedfccd9aa45c76f 2024-11-17T22:41:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/e26a835c93d04774a128db98d7b07e33 is 50, key is test_row_0/B:col10/1731883260451/Put/seqid=0 2024-11-17T22:41:01,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742118_1294 (size=12301) 2024-11-17T22:41:01,401 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/e26a835c93d04774a128db98d7b07e33 2024-11-17T22:41:01,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/9cc8bdddf09e4d3a8f9ed0597aa619f4 is 50, key is test_row_0/C:col10/1731883260451/Put/seqid=0 2024-11-17T22:41:01,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742119_1295 (size=12301) 2024-11-17T22:41:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T22:41:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:01,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:01,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883321619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883321619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883321620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883321621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883321622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883321726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883321727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883321727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883321737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883321737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,748 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/92dcb3fcd66f48a6b100248db8f1dba1 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/92dcb3fcd66f48a6b100248db8f1dba1 2024-11-17T22:41:01,752 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/B of ad1461c99ce1acccd0c930b472c9a7c1 into 92dcb3fcd66f48a6b100248db8f1dba1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:01,752 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:01,752 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/B, priority=13, startTime=1731883261300; duration=0sec 2024-11-17T22:41:01,752 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:01,752 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:B 2024-11-17T22:41:01,753 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:01,754 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:01,754 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/C is initiating minor compaction (all files) 2024-11-17T22:41:01,754 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/C in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:01,754 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4bc87ef0be144160a1d4c23cfc1fe3f4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/a3d49aea18cb4598b1b43da62a84acaf, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/38ab749d024f41ffb8897d14c2219f91] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=36.7 K 2024-11-17T22:41:01,754 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bc87ef0be144160a1d4c23cfc1fe3f4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731883258290 2024-11-17T22:41:01,754 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting a3d49aea18cb4598b1b43da62a84acaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731883258645 2024-11-17T22:41:01,755 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ab749d024f41ffb8897d14c2219f91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731883259806 2024-11-17T22:41:01,761 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/f18062bdf035483d8362ca80fae4f81c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/f18062bdf035483d8362ca80fae4f81c 2024-11-17T22:41:01,761 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:01,762 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/f9419e8552b344a69df488789bd50217 is 50, key is test_row_0/C:col10/1731883259806/Put/seqid=0 2024-11-17T22:41:01,768 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/A of ad1461c99ce1acccd0c930b472c9a7c1 into f18062bdf035483d8362ca80fae4f81c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:01,768 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:01,768 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/A, priority=13, startTime=1731883261300; duration=0sec 2024-11-17T22:41:01,768 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:01,768 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:A 2024-11-17T22:41:01,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742120_1296 (size=13051) 2024-11-17T22:41:01,787 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/f9419e8552b344a69df488789bd50217 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f9419e8552b344a69df488789bd50217 2024-11-17T22:41:01,793 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/C of ad1461c99ce1acccd0c930b472c9a7c1 into f9419e8552b344a69df488789bd50217(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:01,793 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:01,793 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/C, priority=13, startTime=1731883261300; duration=0sec 2024-11-17T22:41:01,794 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:01,794 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:C 2024-11-17T22:41:01,834 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/9cc8bdddf09e4d3a8f9ed0597aa619f4 2024-11-17T22:41:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/03662fa777b34685bedfccd9aa45c76f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/03662fa777b34685bedfccd9aa45c76f 2024-11-17T22:41:01,843 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/03662fa777b34685bedfccd9aa45c76f, entries=150, sequenceid=330, filesize=12.0 K 2024-11-17T22:41:01,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/e26a835c93d04774a128db98d7b07e33 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e26a835c93d04774a128db98d7b07e33 2024-11-17T22:41:01,847 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e26a835c93d04774a128db98d7b07e33, entries=150, sequenceid=330, filesize=12.0 K 2024-11-17T22:41:01,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/9cc8bdddf09e4d3a8f9ed0597aa619f4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/9cc8bdddf09e4d3a8f9ed0597aa619f4 2024-11-17T22:41:01,851 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/9cc8bdddf09e4d3a8f9ed0597aa619f4, entries=150, sequenceid=330, filesize=12.0 K 2024-11-17T22:41:01,852 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ad1461c99ce1acccd0c930b472c9a7c1 in 502ms, sequenceid=330, compaction requested=false 2024-11-17T22:41:01,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:01,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:01,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-17T22:41:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-17T22:41:01,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-17T22:41:01,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 964 msec 2024-11-17T22:41:01,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 968 msec 2024-11-17T22:41:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:01,940 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:01,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T22:41:01,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/b61c1ca958cc469999789342c488d4ed is 50, key is test_row_0/A:col10/1731883261621/Put/seqid=0 2024-11-17T22:41:01,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742121_1297 (size=14741) 2024-11-17T22:41:01,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883321951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883321951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883321952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883321956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883321956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T22:41:02,027 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:02,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-17T22:41:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T22:41:02,030 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:02,030 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:02,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:02,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883322067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883322067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883322067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883322067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883322067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T22:41:02,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-17T22:41:02,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:02,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883322276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883322276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883322277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883322277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883322277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-17T22:41:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T22:41:02,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/b61c1ca958cc469999789342c488d4ed 2024-11-17T22:41:02,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/f71e2d5e9bde47cfbda7a4b024fde94a is 50, key is test_row_0/B:col10/1731883261621/Put/seqid=0 2024-11-17T22:41:02,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742122_1298 (size=12301) 2024-11-17T22:41:02,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-17T22:41:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883322586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883322587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883322587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883322587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883322587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:02,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-17T22:41:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T22:41:02,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/f71e2d5e9bde47cfbda7a4b024fde94a 2024-11-17T22:41:02,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/348016686e034ca3a76f97af90089705 is 50, key is test_row_0/C:col10/1731883261621/Put/seqid=0 2024-11-17T22:41:02,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742123_1299 (size=12301) 2024-11-17T22:41:02,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-17T22:41:02,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:02,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-17T22:41:02,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:02,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:02,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:03,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883323097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883323097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883323097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:03,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883323097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:03,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883323097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:03,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-17T22:41:03,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:03,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:03,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:03,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:03,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T22:41:03,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/348016686e034ca3a76f97af90089705 2024-11-17T22:41:03,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/b61c1ca958cc469999789342c488d4ed as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/b61c1ca958cc469999789342c488d4ed 2024-11-17T22:41:03,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/b61c1ca958cc469999789342c488d4ed, entries=200, sequenceid=357, filesize=14.4 K 2024-11-17T22:41:03,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/f71e2d5e9bde47cfbda7a4b024fde94a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/f71e2d5e9bde47cfbda7a4b024fde94a 2024-11-17T22:41:03,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/f71e2d5e9bde47cfbda7a4b024fde94a, entries=150, sequenceid=357, filesize=12.0 K 2024-11-17T22:41:03,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/348016686e034ca3a76f97af90089705 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/348016686e034ca3a76f97af90089705 2024-11-17T22:41:03,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/348016686e034ca3a76f97af90089705, entries=150, sequenceid=357, filesize=12.0 K 2024-11-17T22:41:03,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for ad1461c99ce1acccd0c930b472c9a7c1 in 1257ms, sequenceid=357, compaction requested=true 2024-11-17T22:41:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:03,198 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:03,198 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:03,199 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:03,199 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:03,199 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/A is initiating minor compaction (all files) 2024-11-17T22:41:03,199 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/B is initiating minor compaction (all files) 2024-11-17T22:41:03,199 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/A in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:03,199 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/B in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:03,199 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/f18062bdf035483d8362ca80fae4f81c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/03662fa777b34685bedfccd9aa45c76f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/b61c1ca958cc469999789342c488d4ed] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=39.2 K 2024-11-17T22:41:03,199 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/92dcb3fcd66f48a6b100248db8f1dba1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e26a835c93d04774a128db98d7b07e33, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/f71e2d5e9bde47cfbda7a4b024fde94a] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=36.8 K 2024-11-17T22:41:03,199 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting f18062bdf035483d8362ca80fae4f81c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731883259806 2024-11-17T22:41:03,199 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 92dcb3fcd66f48a6b100248db8f1dba1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731883259806 2024-11-17T22:41:03,199 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting e26a835c93d04774a128db98d7b07e33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731883260451 2024-11-17T22:41:03,199 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03662fa777b34685bedfccd9aa45c76f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731883260451 2024-11-17T22:41:03,200 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting b61c1ca958cc469999789342c488d4ed, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731883261621 2024-11-17T22:41:03,200 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting f71e2d5e9bde47cfbda7a4b024fde94a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731883261621 2024-11-17T22:41:03,207 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#B#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:03,207 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/e04810c6e48e4a9aafe1f7c6b37d2f82 is 50, key is test_row_0/B:col10/1731883261621/Put/seqid=0 2024-11-17T22:41:03,210 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#A#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:03,210 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/33c10a2f4b6f4748a4e86419ac3d06d6 is 50, key is test_row_0/A:col10/1731883261621/Put/seqid=0 2024-11-17T22:41:03,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742124_1300 (size=13153) 2024-11-17T22:41:03,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742125_1301 (size=13153) 2024-11-17T22:41:03,220 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/e04810c6e48e4a9aafe1f7c6b37d2f82 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e04810c6e48e4a9aafe1f7c6b37d2f82 2024-11-17T22:41:03,223 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/33c10a2f4b6f4748a4e86419ac3d06d6 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/33c10a2f4b6f4748a4e86419ac3d06d6 2024-11-17T22:41:03,227 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/B of ad1461c99ce1acccd0c930b472c9a7c1 into e04810c6e48e4a9aafe1f7c6b37d2f82(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:03,227 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:03,227 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/B, priority=13, startTime=1731883263198; duration=0sec 2024-11-17T22:41:03,227 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:03,227 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:B 2024-11-17T22:41:03,227 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:03,228 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/A of ad1461c99ce1acccd0c930b472c9a7c1 into 33c10a2f4b6f4748a4e86419ac3d06d6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:03,228 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:03,228 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/A, priority=13, startTime=1731883263198; duration=0sec 2024-11-17T22:41:03,228 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:03,229 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:A 2024-11-17T22:41:03,229 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:03,229 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/C is initiating minor compaction (all files) 2024-11-17T22:41:03,229 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/C in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:03,229 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f9419e8552b344a69df488789bd50217, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/9cc8bdddf09e4d3a8f9ed0597aa619f4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/348016686e034ca3a76f97af90089705] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=36.8 K 2024-11-17T22:41:03,229 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting f9419e8552b344a69df488789bd50217, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731883259806 2024-11-17T22:41:03,230 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cc8bdddf09e4d3a8f9ed0597aa619f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731883260451 2024-11-17T22:41:03,230 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 348016686e034ca3a76f97af90089705, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731883261621 2024-11-17T22:41:03,251 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#C#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:03,252 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/03b2a81d773242588f4b2bc3bf0ad7e8 is 50, key is test_row_0/C:col10/1731883261621/Put/seqid=0 2024-11-17T22:41:03,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742126_1302 (size=13153) 2024-11-17T22:41:03,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-17T22:41:03,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:03,257 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:41:03,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/592eec25accf4244aea0188b089e092c is 50, key is test_row_0/A:col10/1731883261955/Put/seqid=0 2024-11-17T22:41:03,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742127_1303 (size=12301) 2024-11-17T22:41:03,662 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/03b2a81d773242588f4b2bc3bf0ad7e8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/03b2a81d773242588f4b2bc3bf0ad7e8 2024-11-17T22:41:03,668 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/C of ad1461c99ce1acccd0c930b472c9a7c1 into 03b2a81d773242588f4b2bc3bf0ad7e8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:03,668 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:03,668 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/C, priority=13, startTime=1731883263198; duration=0sec 2024-11-17T22:41:03,668 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:03,668 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:C 2024-11-17T22:41:03,677 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/592eec25accf4244aea0188b089e092c 2024-11-17T22:41:03,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/4e83d6fda2404dee95ad249825427642 is 50, key is test_row_0/B:col10/1731883261955/Put/seqid=0 2024-11-17T22:41:03,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742128_1304 (size=12301) 2024-11-17T22:41:03,692 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/4e83d6fda2404dee95ad249825427642 2024-11-17T22:41:03,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/3391b1cb877b474d89ceaf351277a925 is 50, key is test_row_0/C:col10/1731883261955/Put/seqid=0 2024-11-17T22:41:03,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742129_1305 (size=12301) 2024-11-17T22:41:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:04,113 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/3391b1cb877b474d89ceaf351277a925 2024-11-17T22:41:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/592eec25accf4244aea0188b089e092c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/592eec25accf4244aea0188b089e092c 2024-11-17T22:41:04,124 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/592eec25accf4244aea0188b089e092c, entries=150, sequenceid=372, filesize=12.0 K 2024-11-17T22:41:04,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/4e83d6fda2404dee95ad249825427642 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4e83d6fda2404dee95ad249825427642 2024-11-17T22:41:04,130 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4e83d6fda2404dee95ad249825427642, entries=150, sequenceid=372, filesize=12.0 K 2024-11-17T22:41:04,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/3391b1cb877b474d89ceaf351277a925 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3391b1cb877b474d89ceaf351277a925 2024-11-17T22:41:04,138 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3391b1cb877b474d89ceaf351277a925, entries=150, sequenceid=372, filesize=12.0 K 2024-11-17T22:41:04,139 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=127.47 KB/130530 for ad1461c99ce1acccd0c930b472c9a7c1 in 882ms, sequenceid=372, compaction requested=false 2024-11-17T22:41:04,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:04,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:04,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-17T22:41:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:04,140 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-17T22:41:04,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:41:04,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-17T22:41:04,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1110 sec 2024-11-17T22:41:04,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 2.1160 sec 2024-11-17T22:41:04,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/39b81df1652a48ffaa38458bee7de2ca is 50, key is test_row_0/A:col10/1731883264140/Put/seqid=0 2024-11-17T22:41:04,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742130_1306 (size=17181) 2024-11-17T22:41:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T22:41:04,178 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:04,178 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees 2024-11-17T22:41:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T22:41:04,180 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:04,181 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:04,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:04,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883324149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883324184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883324184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883324185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883324185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T22:41:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883324297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883324297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883324297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883324297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883324297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-17T22:41:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:04,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-17T22:41:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:04,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T22:41:04,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883324507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883324507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883324507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883324507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883324507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/39b81df1652a48ffaa38458bee7de2ca 2024-11-17T22:41:04,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b3968eec34764eefb1b001e69eb308c4 is 50, key is test_row_0/B:col10/1731883264140/Put/seqid=0 2024-11-17T22:41:04,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742131_1307 (size=12301) 2024-11-17T22:41:04,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-17T22:41:04,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:04,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:04,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:04,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-17T22:41:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:04,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T22:41:04,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883324817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883324817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883324817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883324817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883324817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:04,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-17T22:41:04,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:04,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:04,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:04,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:05,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b3968eec34764eefb1b001e69eb308c4 2024-11-17T22:41:05,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/57fda99b772849d1b35e7844ae24c443 is 50, key is test_row_0/C:col10/1731883264140/Put/seqid=0 2024-11-17T22:41:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742132_1308 (size=12301) 2024-11-17T22:41:05,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/57fda99b772849d1b35e7844ae24c443 2024-11-17T22:41:05,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/39b81df1652a48ffaa38458bee7de2ca as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/39b81df1652a48ffaa38458bee7de2ca 2024-11-17T22:41:05,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/39b81df1652a48ffaa38458bee7de2ca, entries=250, sequenceid=399, filesize=16.8 K 2024-11-17T22:41:05,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/b3968eec34764eefb1b001e69eb308c4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b3968eec34764eefb1b001e69eb308c4 2024-11-17T22:41:05,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b3968eec34764eefb1b001e69eb308c4, entries=150, sequenceid=399, filesize=12.0 K 2024-11-17T22:41:05,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/57fda99b772849d1b35e7844ae24c443 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/57fda99b772849d1b35e7844ae24c443 2024-11-17T22:41:05,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/57fda99b772849d1b35e7844ae24c443, entries=150, sequenceid=399, filesize=12.0 K 2024-11-17T22:41:05,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ad1461c99ce1acccd0c930b472c9a7c1 in 933ms, sequenceid=399, compaction requested=true 2024-11-17T22:41:05,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:05,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:05,074 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:05,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:05,074 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:05,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:05,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:05,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:05,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:05,076 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:05,077 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/A is initiating minor compaction (all files) 2024-11-17T22:41:05,077 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/A in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:05,077 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/33c10a2f4b6f4748a4e86419ac3d06d6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/592eec25accf4244aea0188b089e092c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/39b81df1652a48ffaa38458bee7de2ca] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=41.6 K 2024-11-17T22:41:05,077 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:05,077 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/B is initiating minor compaction (all files) 2024-11-17T22:41:05,077 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/B in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:05,077 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e04810c6e48e4a9aafe1f7c6b37d2f82, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4e83d6fda2404dee95ad249825427642, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b3968eec34764eefb1b001e69eb308c4] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=36.9 K 2024-11-17T22:41:05,078 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting e04810c6e48e4a9aafe1f7c6b37d2f82, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731883261621 2024-11-17T22:41:05,078 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33c10a2f4b6f4748a4e86419ac3d06d6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731883261621 2024-11-17T22:41:05,079 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e83d6fda2404dee95ad249825427642, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731883261945 2024-11-17T22:41:05,079 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 592eec25accf4244aea0188b089e092c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731883261945 2024-11-17T22:41:05,079 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b3968eec34764eefb1b001e69eb308c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731883264138 2024-11-17T22:41:05,079 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b81df1652a48ffaa38458bee7de2ca, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731883264134 2024-11-17T22:41:05,095 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#B#compaction#266 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:05,095 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/aeb3b972bc6942b89279d4941d69e9c0 is 50, key is test_row_0/B:col10/1731883264140/Put/seqid=0 2024-11-17T22:41:05,102 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#A#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:05,102 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/a09205edb3f749e78149f9ce1bb830d4 is 50, key is test_row_0/A:col10/1731883264140/Put/seqid=0 2024-11-17T22:41:05,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-17T22:41:05,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:05,114 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T22:41:05,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/c51c455fdf414cd6959f531602869daf is 50, key is test_row_1/A:col10/1731883264184/Put/seqid=0 2024-11-17T22:41:05,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742133_1309 (size=13255) 2024-11-17T22:41:05,146 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/aeb3b972bc6942b89279d4941d69e9c0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/aeb3b972bc6942b89279d4941d69e9c0 2024-11-17T22:41:05,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742134_1310 (size=13255) 2024-11-17T22:41:05,154 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/B of ad1461c99ce1acccd0c930b472c9a7c1 into aeb3b972bc6942b89279d4941d69e9c0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:05,154 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:05,154 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/B, priority=13, startTime=1731883265074; duration=0sec 2024-11-17T22:41:05,155 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:05,155 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:B 2024-11-17T22:41:05,155 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:05,156 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:05,156 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/C is initiating minor compaction (all files) 2024-11-17T22:41:05,156 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/C in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:05,157 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/03b2a81d773242588f4b2bc3bf0ad7e8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3391b1cb877b474d89ceaf351277a925, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/57fda99b772849d1b35e7844ae24c443] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=36.9 K 2024-11-17T22:41:05,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742135_1311 (size=9857) 2024-11-17T22:41:05,158 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/c51c455fdf414cd6959f531602869daf 2024-11-17T22:41:05,159 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/a09205edb3f749e78149f9ce1bb830d4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/a09205edb3f749e78149f9ce1bb830d4 2024-11-17T22:41:05,159 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 03b2a81d773242588f4b2bc3bf0ad7e8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731883261621 2024-11-17T22:41:05,159 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 3391b1cb877b474d89ceaf351277a925, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731883261945 2024-11-17T22:41:05,160 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 57fda99b772849d1b35e7844ae24c443, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731883264138 2024-11-17T22:41:05,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/7713ecc34cab4cc699d484238aac4024 is 50, key is test_row_1/B:col10/1731883264184/Put/seqid=0 2024-11-17T22:41:05,168 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/A of ad1461c99ce1acccd0c930b472c9a7c1 into a09205edb3f749e78149f9ce1bb830d4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:05,168 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:05,169 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/A, priority=13, startTime=1731883265074; duration=0sec 2024-11-17T22:41:05,169 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:05,169 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:A 2024-11-17T22:41:05,179 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:05,180 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/f46feba8224a47318d8a0fcaa8d13e4b is 50, key is test_row_0/C:col10/1731883264140/Put/seqid=0 2024-11-17T22:41:05,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742136_1312 (size=9857) 2024-11-17T22:41:05,181 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/7713ecc34cab4cc699d484238aac4024 2024-11-17T22:41:05,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742137_1313 (size=13255) 2024-11-17T22:41:05,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/139256c443e7400cbeb4efae21412166 is 50, key is test_row_1/C:col10/1731883264184/Put/seqid=0 2024-11-17T22:41:05,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742138_1314 (size=9857) 2024-11-17T22:41:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T22:41:05,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:05,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883325356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883325356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883325357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883325359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883325359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883325467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883325467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883325467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883325467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883325478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,592 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/f46feba8224a47318d8a0fcaa8d13e4b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f46feba8224a47318d8a0fcaa8d13e4b 2024-11-17T22:41:05,604 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/C of ad1461c99ce1acccd0c930b472c9a7c1 into f46feba8224a47318d8a0fcaa8d13e4b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:05,604 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:05,604 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/C, priority=13, startTime=1731883265075; duration=0sec 2024-11-17T22:41:05,605 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:05,605 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:C 2024-11-17T22:41:05,610 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/139256c443e7400cbeb4efae21412166 2024-11-17T22:41:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/c51c455fdf414cd6959f531602869daf as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/c51c455fdf414cd6959f531602869daf 2024-11-17T22:41:05,623 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/c51c455fdf414cd6959f531602869daf, entries=100, sequenceid=409, filesize=9.6 K 2024-11-17T22:41:05,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/7713ecc34cab4cc699d484238aac4024 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/7713ecc34cab4cc699d484238aac4024 2024-11-17T22:41:05,634 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/7713ecc34cab4cc699d484238aac4024, entries=100, sequenceid=409, filesize=9.6 K 2024-11-17T22:41:05,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/139256c443e7400cbeb4efae21412166 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/139256c443e7400cbeb4efae21412166 2024-11-17T22:41:05,651 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/139256c443e7400cbeb4efae21412166, entries=100, sequenceid=409, filesize=9.6 K 2024-11-17T22:41:05,652 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for ad1461c99ce1acccd0c930b472c9a7c1 in 539ms, sequenceid=409, compaction requested=false 2024-11-17T22:41:05,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:05,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:05,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=93 2024-11-17T22:41:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=93 2024-11-17T22:41:05,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-17T22:41:05,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-11-17T22:41:05,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees in 1.4790 sec 2024-11-17T22:41:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:05,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-17T22:41:05,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/04386a80f3f24d4281b2a8a71a66fbb1 is 50, key is test_row_0/A:col10/1731883265358/Put/seqid=0 2024-11-17T22:41:05,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883325687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883325688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883325688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883325690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883325690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742139_1315 (size=17181) 2024-11-17T22:41:05,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883325797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883325797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883325797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883325798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883325997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883326006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883326007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883326007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883326009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/04386a80f3f24d4281b2a8a71a66fbb1 2024-11-17T22:41:06,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/5599248bfba14876ab599994363c23fb is 50, key is test_row_0/B:col10/1731883265358/Put/seqid=0 2024-11-17T22:41:06,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742140_1316 (size=12301) 2024-11-17T22:41:06,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/5599248bfba14876ab599994363c23fb 2024-11-17T22:41:06,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4f565bb4aba74377affc26d8b2d781e1 is 50, key is test_row_0/C:col10/1731883265358/Put/seqid=0 2024-11-17T22:41:06,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742141_1317 (size=12301) 2024-11-17T22:41:06,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4f565bb4aba74377affc26d8b2d781e1 2024-11-17T22:41:06,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/04386a80f3f24d4281b2a8a71a66fbb1 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/04386a80f3f24d4281b2a8a71a66fbb1 2024-11-17T22:41:06,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/04386a80f3f24d4281b2a8a71a66fbb1, entries=250, sequenceid=440, filesize=16.8 K 2024-11-17T22:41:06,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/5599248bfba14876ab599994363c23fb as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5599248bfba14876ab599994363c23fb 2024-11-17T22:41:06,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5599248bfba14876ab599994363c23fb, entries=150, sequenceid=440, filesize=12.0 K 2024-11-17T22:41:06,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/4f565bb4aba74377affc26d8b2d781e1 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f565bb4aba74377affc26d8b2d781e1 2024-11-17T22:41:06,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f565bb4aba74377affc26d8b2d781e1, entries=150, sequenceid=440, filesize=12.0 K 2024-11-17T22:41:06,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for ad1461c99ce1acccd0c930b472c9a7c1 in 550ms, sequenceid=440, compaction requested=true 2024-11-17T22:41:06,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:06,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:06,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:06,231 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:06,231 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:06,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:06,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:06,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:06,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:06,233 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:06,233 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/B is initiating minor compaction (all files) 2024-11-17T22:41:06,233 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40293 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:06,233 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/B in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:06,233 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/A is initiating minor compaction (all files) 2024-11-17T22:41:06,233 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/A in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:06,233 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/aeb3b972bc6942b89279d4941d69e9c0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/7713ecc34cab4cc699d484238aac4024, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5599248bfba14876ab599994363c23fb] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=34.6 K 2024-11-17T22:41:06,233 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/a09205edb3f749e78149f9ce1bb830d4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/c51c455fdf414cd6959f531602869daf, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/04386a80f3f24d4281b2a8a71a66fbb1] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=39.3 K 2024-11-17T22:41:06,234 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting a09205edb3f749e78149f9ce1bb830d4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731883264138 2024-11-17T22:41:06,234 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting aeb3b972bc6942b89279d4941d69e9c0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731883264138 2024-11-17T22:41:06,234 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting c51c455fdf414cd6959f531602869daf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731883264153 2024-11-17T22:41:06,234 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 7713ecc34cab4cc699d484238aac4024, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731883264153 2024-11-17T22:41:06,235 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04386a80f3f24d4281b2a8a71a66fbb1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731883265352 2024-11-17T22:41:06,235 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 5599248bfba14876ab599994363c23fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731883265352 2024-11-17T22:41:06,247 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#A#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:06,247 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/8eb81969037e4f1f91e813a2cc990a50 is 50, key is test_row_0/A:col10/1731883265358/Put/seqid=0 2024-11-17T22:41:06,247 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:06,248 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/a46b15504772477dab45ba1778ae588c is 50, key is test_row_0/B:col10/1731883265358/Put/seqid=0 2024-11-17T22:41:06,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742142_1318 (size=13357) 2024-11-17T22:41:06,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742143_1319 (size=13357) 2024-11-17T22:41:06,278 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/a46b15504772477dab45ba1778ae588c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/a46b15504772477dab45ba1778ae588c 2024-11-17T22:41:06,285 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/B of ad1461c99ce1acccd0c930b472c9a7c1 into a46b15504772477dab45ba1778ae588c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:06,285 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:06,286 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/B, priority=13, startTime=1731883266231; duration=0sec 2024-11-17T22:41:06,286 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:06,286 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:B 2024-11-17T22:41:06,286 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:06,287 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:06,288 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/C is initiating minor compaction (all files) 2024-11-17T22:41:06,288 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/C in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:06,288 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f46feba8224a47318d8a0fcaa8d13e4b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/139256c443e7400cbeb4efae21412166, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f565bb4aba74377affc26d8b2d781e1] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=34.6 K 2024-11-17T22:41:06,288 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting f46feba8224a47318d8a0fcaa8d13e4b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731883264138 2024-11-17T22:41:06,289 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 139256c443e7400cbeb4efae21412166, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731883264153 2024-11-17T22:41:06,289 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f565bb4aba74377affc26d8b2d781e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731883265352 2024-11-17T22:41:06,301 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#C#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:06,301 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/241aa18ea2a54419bd62a4591498be92 is 50, key is test_row_0/C:col10/1731883265358/Put/seqid=0 2024-11-17T22:41:06,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742144_1320 (size=13357) 2024-11-17T22:41:06,322 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/241aa18ea2a54419bd62a4591498be92 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/241aa18ea2a54419bd62a4591498be92 2024-11-17T22:41:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:06,330 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:06,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:41:06,330 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/C of ad1461c99ce1acccd0c930b472c9a7c1 into 241aa18ea2a54419bd62a4591498be92(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:06,330 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:06,331 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/C, priority=13, startTime=1731883266231; duration=0sec 2024-11-17T22:41:06,332 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:06,332 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:C 2024-11-17T22:41:06,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/06a1ec87e9034be292bd718f4a75eb3d is 50, key is test_row_0/A:col10/1731883266329/Put/seqid=0 2024-11-17T22:41:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T22:41:06,337 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:06,338 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees 2024-11-17T22:41:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T22:41:06,340 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=94, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:06,341 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=94, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:06,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:06,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742145_1321 (size=17181) 2024-11-17T22:41:06,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883326372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883326373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883326373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883326375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T22:41:06,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883326487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883326487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883326487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883326487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T22:41:06,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:06,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:06,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:06,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883326507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T22:41:06,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:06,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:06,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:06,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T22:41:06,671 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/8eb81969037e4f1f91e813a2cc990a50 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/8eb81969037e4f1f91e813a2cc990a50 2024-11-17T22:41:06,677 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/A of ad1461c99ce1acccd0c930b472c9a7c1 into 8eb81969037e4f1f91e813a2cc990a50(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:06,677 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:06,677 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/A, priority=13, startTime=1731883266230; duration=0sec 2024-11-17T22:41:06,678 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:06,678 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:A 2024-11-17T22:41:06,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883326697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883326697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883326697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:06,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883326698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:06,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:06,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:06,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/06a1ec87e9034be292bd718f4a75eb3d 2024-11-17T22:41:06,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/19c2c51d3f6a45f9a6a5e3a21dfab478 is 50, key is test_row_0/B:col10/1731883266329/Put/seqid=0 2024-11-17T22:41:06,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742146_1322 (size=12301) 2024-11-17T22:41:06,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T22:41:06,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:06,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:06,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:06,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T22:41:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:06,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T22:41:07,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883327007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:07,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:07,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883327007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883327008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:07,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883327008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:07,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T22:41:07,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:07,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:07,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:07,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/19c2c51d3f6a45f9a6a5e3a21dfab478 2024-11-17T22:41:07,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/0a5e6c6d10264c478f6a2b0f0c8cc46a is 50, key is test_row_0/C:col10/1731883266329/Put/seqid=0 2024-11-17T22:41:07,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742147_1323 (size=12301) 2024-11-17T22:41:07,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T22:41:07,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:07,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:07,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:07,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T22:41:07,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:07,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:07,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:07,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T22:41:07,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883327517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:07,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:07,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883327526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:07,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883327527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:07,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883327527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883327527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:07,530 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in ad1461c99ce1acccd0c930b472c9a7c1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:07,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:07,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:07,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T22:41:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:07,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:07,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:07,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:07,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/0a5e6c6d10264c478f6a2b0f0c8cc46a 2024-11-17T22:41:07,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/06a1ec87e9034be292bd718f4a75eb3d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/06a1ec87e9034be292bd718f4a75eb3d 2024-11-17T22:41:07,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/06a1ec87e9034be292bd718f4a75eb3d, entries=250, sequenceid=454, filesize=16.8 K 2024-11-17T22:41:07,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/19c2c51d3f6a45f9a6a5e3a21dfab478 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/19c2c51d3f6a45f9a6a5e3a21dfab478 2024-11-17T22:41:07,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/19c2c51d3f6a45f9a6a5e3a21dfab478, entries=150, sequenceid=454, filesize=12.0 K 2024-11-17T22:41:07,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/0a5e6c6d10264c478f6a2b0f0c8cc46a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0a5e6c6d10264c478f6a2b0f0c8cc46a 2024-11-17T22:41:07,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0a5e6c6d10264c478f6a2b0f0c8cc46a, entries=150, sequenceid=454, filesize=12.0 K 2024-11-17T22:41:07,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ad1461c99ce1acccd0c930b472c9a7c1 in 1262ms, sequenceid=454, compaction requested=false 2024-11-17T22:41:07,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:07,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T22:41:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:07,718 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:41:07,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/a2a9dfc1d79b428ebc3975961845abf9 is 50, key is test_row_0/A:col10/1731883266370/Put/seqid=0 2024-11-17T22:41:07,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742148_1324 (size=12301) 2024-11-17T22:41:08,127 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/a2a9dfc1d79b428ebc3975961845abf9 2024-11-17T22:41:08,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/0c35b70ee2a9418fb22b586c4dc5b97c is 50, key is test_row_0/B:col10/1731883266370/Put/seqid=0 2024-11-17T22:41:08,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742149_1325 (size=12301) 2024-11-17T22:41:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T22:41:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:08,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. as already flushing 2024-11-17T22:41:08,550 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/0c35b70ee2a9418fb22b586c4dc5b97c 2024-11-17T22:41:08,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:08,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883328546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883328547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:08,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:08,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:08,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55874 deadline: 1731883328547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:08,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55894 deadline: 1731883328547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:08,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:08,551 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in ad1461c99ce1acccd0c930b472c9a7c1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:08,551 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in ad1461c99ce1acccd0c930b472c9a7c1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:08,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/628651909d4341e4bc0d86121485382d is 50, key is test_row_0/C:col10/1731883266370/Put/seqid=0 2024-11-17T22:41:08,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742150_1326 (size=12301) 2024-11-17T22:41:08,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883328657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:08,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883328657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:08,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:08,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883328867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883328867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:08,961 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/628651909d4341e4bc0d86121485382d 2024-11-17T22:41:08,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/a2a9dfc1d79b428ebc3975961845abf9 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/a2a9dfc1d79b428ebc3975961845abf9 2024-11-17T22:41:08,968 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/a2a9dfc1d79b428ebc3975961845abf9, entries=150, sequenceid=479, filesize=12.0 K 2024-11-17T22:41:08,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/0c35b70ee2a9418fb22b586c4dc5b97c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/0c35b70ee2a9418fb22b586c4dc5b97c 2024-11-17T22:41:08,973 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/0c35b70ee2a9418fb22b586c4dc5b97c, entries=150, sequenceid=479, filesize=12.0 K 2024-11-17T22:41:08,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/628651909d4341e4bc0d86121485382d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/628651909d4341e4bc0d86121485382d 2024-11-17T22:41:08,979 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/628651909d4341e4bc0d86121485382d, entries=150, sequenceid=479, filesize=12.0 K 2024-11-17T22:41:08,980 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ad1461c99ce1acccd0c930b472c9a7c1 in 1262ms, sequenceid=479, compaction requested=true 2024-11-17T22:41:08,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:08,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:08,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=95 2024-11-17T22:41:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=95 2024-11-17T22:41:08,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-17T22:41:08,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6400 sec 2024-11-17T22:41:08,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees in 2.6460 sec 2024-11-17T22:41:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:09,179 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:09,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:41:09,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/dc37d09a44d84048adc90d9be7f74f8e is 50, key is test_row_0/A:col10/1731883268545/Put/seqid=0 2024-11-17T22:41:09,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742151_1327 (size=14741) 2024-11-17T22:41:09,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:09,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883329226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883329225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883329337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:09,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883329337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:09,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883329547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:09,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883329547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:09,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55926 deadline: 1731883329556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:09,559 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in ad1461c99ce1acccd0c930b472c9a7c1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:09,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/dc37d09a44d84048adc90d9be7f74f8e 2024-11-17T22:41:09,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/857c3d635431439c8120d05a72a639a6 is 50, key is test_row_0/B:col10/1731883268545/Put/seqid=0 2024-11-17T22:41:09,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742152_1328 (size=12301) 2024-11-17T22:41:09,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55888 deadline: 1731883329856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:09,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55918 deadline: 1731883329857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:10,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/857c3d635431439c8120d05a72a639a6 2024-11-17T22:41:10,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/f6dab66b3450442180f3b67f8a668805 is 50, key is test_row_0/C:col10/1731883268545/Put/seqid=0 2024-11-17T22:41:10,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742153_1329 (size=12301) 2024-11-17T22:41:10,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/f6dab66b3450442180f3b67f8a668805 2024-11-17T22:41:10,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/dc37d09a44d84048adc90d9be7f74f8e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/dc37d09a44d84048adc90d9be7f74f8e 2024-11-17T22:41:10,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/dc37d09a44d84048adc90d9be7f74f8e, entries=200, sequenceid=492, filesize=14.4 K 2024-11-17T22:41:10,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/857c3d635431439c8120d05a72a639a6 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/857c3d635431439c8120d05a72a639a6 2024-11-17T22:41:10,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/857c3d635431439c8120d05a72a639a6, entries=150, sequenceid=492, filesize=12.0 K 2024-11-17T22:41:10,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/f6dab66b3450442180f3b67f8a668805 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f6dab66b3450442180f3b67f8a668805 2024-11-17T22:41:10,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f6dab66b3450442180f3b67f8a668805, entries=150, sequenceid=492, filesize=12.0 K 2024-11-17T22:41:10,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ad1461c99ce1acccd0c930b472c9a7c1 in 854ms, sequenceid=492, compaction requested=true 2024-11-17T22:41:10,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:10,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:10,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:10,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:10,034 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:41:10,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:10,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1461c99ce1acccd0c930b472c9a7c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:10,034 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:41:10,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:10,035 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:41:10,035 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:41:10,035 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/B is initiating minor compaction (all files) 2024-11-17T22:41:10,035 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/A is initiating minor compaction (all files) 2024-11-17T22:41:10,035 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/B in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:10,035 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/A in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:10,035 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/8eb81969037e4f1f91e813a2cc990a50, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/06a1ec87e9034be292bd718f4a75eb3d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/a2a9dfc1d79b428ebc3975961845abf9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/dc37d09a44d84048adc90d9be7f74f8e] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=56.2 K 2024-11-17T22:41:10,035 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/a46b15504772477dab45ba1778ae588c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/19c2c51d3f6a45f9a6a5e3a21dfab478, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/0c35b70ee2a9418fb22b586c4dc5b97c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/857c3d635431439c8120d05a72a639a6] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=49.1 K 2024-11-17T22:41:10,036 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eb81969037e4f1f91e813a2cc990a50, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731883265352 2024-11-17T22:41:10,036 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting a46b15504772477dab45ba1778ae588c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731883265352 2024-11-17T22:41:10,036 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06a1ec87e9034be292bd718f4a75eb3d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731883265686 2024-11-17T22:41:10,036 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c2c51d3f6a45f9a6a5e3a21dfab478, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731883265686 2024-11-17T22:41:10,036 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2a9dfc1d79b428ebc3975961845abf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731883266370 2024-11-17T22:41:10,036 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c35b70ee2a9418fb22b586c4dc5b97c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731883266370 2024-11-17T22:41:10,037 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc37d09a44d84048adc90d9be7f74f8e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731883268528 2024-11-17T22:41:10,037 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 857c3d635431439c8120d05a72a639a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731883268528 2024-11-17T22:41:10,044 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#B#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:10,044 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/2b3612ee1d384bc880b32687f1167d5c is 50, key is test_row_0/B:col10/1731883268545/Put/seqid=0 2024-11-17T22:41:10,047 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#A#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:10,047 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/d77dbf289ae64693bd8e35dc1c8e5d65 is 50, key is test_row_0/A:col10/1731883268545/Put/seqid=0 2024-11-17T22:41:10,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742155_1331 (size=13493) 2024-11-17T22:41:10,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742154_1330 (size=13493) 2024-11-17T22:41:10,363 INFO [Thread-1061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1061. 2024-11-17T22:41:10,363 DEBUG [Thread-1061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:10,363 DEBUG [Thread-1061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,363 DEBUG [Thread-1061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:10,364 INFO [Thread-1067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1067. 2024-11-17T22:41:10,364 INFO [Thread-1065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1065. 2024-11-17T22:41:10,364 DEBUG [Thread-1067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:10,364 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,364 DEBUG [Thread-1065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:10,364 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,364 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:10,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:10,364 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,367 INFO [Thread-1063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1063. 2024-11-17T22:41:10,367 DEBUG [Thread-1063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:10,367 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,367 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:10,368 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-11-17T22:41:10,368 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:10,368 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,368 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,368 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:10,369 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:10,369 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-11-17T22:41:10,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:41:10,369 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-17T22:41:10,369 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:10,369 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:10,369 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,369 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,369 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,369 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:10,369 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,369 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:10,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/3eac31fb00b44b51b35cac219c9899c3 is 50, key is test_row_0/A:col10/1731883270367/Put/seqid=0 2024-11-17T22:41:10,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742156_1332 (size=12301) 2024-11-17T22:41:10,459 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/d77dbf289ae64693bd8e35dc1c8e5d65 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d77dbf289ae64693bd8e35dc1c8e5d65 2024-11-17T22:41:10,462 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/2b3612ee1d384bc880b32687f1167d5c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2b3612ee1d384bc880b32687f1167d5c 2024-11-17T22:41:10,466 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/A of ad1461c99ce1acccd0c930b472c9a7c1 into d77dbf289ae64693bd8e35dc1c8e5d65(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:10,466 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:10,466 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/A, priority=12, startTime=1731883270034; duration=0sec 2024-11-17T22:41:10,467 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:10,467 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:A 2024-11-17T22:41:10,467 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:41:10,468 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/B of ad1461c99ce1acccd0c930b472c9a7c1 into 2b3612ee1d384bc880b32687f1167d5c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:10,468 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:10,468 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/B, priority=12, startTime=1731883270034; duration=0sec 2024-11-17T22:41:10,468 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:10,468 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:B 2024-11-17T22:41:10,468 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:41:10,468 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): ad1461c99ce1acccd0c930b472c9a7c1/C is initiating minor compaction (all files) 2024-11-17T22:41:10,469 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1461c99ce1acccd0c930b472c9a7c1/C in TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:10,469 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/241aa18ea2a54419bd62a4591498be92, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0a5e6c6d10264c478f6a2b0f0c8cc46a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/628651909d4341e4bc0d86121485382d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f6dab66b3450442180f3b67f8a668805] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp, totalSize=49.1 K 2024-11-17T22:41:10,469 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 241aa18ea2a54419bd62a4591498be92, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731883265352 2024-11-17T22:41:10,470 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a5e6c6d10264c478f6a2b0f0c8cc46a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731883265686 2024-11-17T22:41:10,470 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 628651909d4341e4bc0d86121485382d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731883266370 2024-11-17T22:41:10,470 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6dab66b3450442180f3b67f8a668805, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731883268528 2024-11-17T22:41:10,480 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1461c99ce1acccd0c930b472c9a7c1#C#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:10,480 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/d5722a469d224edbbc7215f58ad26e71 is 50, key is test_row_0/C:col10/1731883268545/Put/seqid=0 2024-11-17T22:41:10,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742157_1333 (size=13493) 2024-11-17T22:41:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T22:41:10,497 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:10,559 INFO [Thread-1050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1050. 2024-11-17T22:41:10,559 DEBUG [Thread-1050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:10,559 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,559 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:10,569 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-17T22:41:10,569 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:10,569 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,569 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:10,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:10,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/3eac31fb00b44b51b35cac219c9899c3 2024-11-17T22:41:10,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/ef5632c3821e42c09b3f6d815ad097c4 is 50, key is test_row_0/B:col10/1731883270367/Put/seqid=0 2024-11-17T22:41:10,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742158_1334 (size=12301) 2024-11-17T22:41:10,893 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/d5722a469d224edbbc7215f58ad26e71 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/d5722a469d224edbbc7215f58ad26e71 2024-11-17T22:41:10,899 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1461c99ce1acccd0c930b472c9a7c1/C of ad1461c99ce1acccd0c930b472c9a7c1 into d5722a469d224edbbc7215f58ad26e71(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:10,899 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:10,899 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., storeName=ad1461c99ce1acccd0c930b472c9a7c1/C, priority=12, startTime=1731883270034; duration=0sec 2024-11-17T22:41:10,899 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:10,899 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1461c99ce1acccd0c930b472c9a7c1:C 2024-11-17T22:41:11,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/ef5632c3821e42c09b3f6d815ad097c4 2024-11-17T22:41:11,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/6976074313e340e0875d1077cae1e572 is 50, key is test_row_0/C:col10/1731883270367/Put/seqid=0 2024-11-17T22:41:11,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742159_1335 (size=12301) 2024-11-17T22:41:11,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/6976074313e340e0875d1077cae1e572 2024-11-17T22:41:11,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/3eac31fb00b44b51b35cac219c9899c3 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/3eac31fb00b44b51b35cac219c9899c3 2024-11-17T22:41:11,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/3eac31fb00b44b51b35cac219c9899c3, entries=150, sequenceid=517, filesize=12.0 K 2024-11-17T22:41:11,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/ef5632c3821e42c09b3f6d815ad097c4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/ef5632c3821e42c09b3f6d815ad097c4 2024-11-17T22:41:11,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/ef5632c3821e42c09b3f6d815ad097c4, entries=150, sequenceid=517, filesize=12.0 K 2024-11-17T22:41:11,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/6976074313e340e0875d1077cae1e572 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/6976074313e340e0875d1077cae1e572 2024-11-17T22:41:11,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/6976074313e340e0875d1077cae1e572, entries=150, sequenceid=517, filesize=12.0 K 2024-11-17T22:41:11,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=13.42 KB/13740 for ad1461c99ce1acccd0c930b472c9a7c1 in 1258ms, sequenceid=517, compaction requested=false 2024-11-17T22:41:11,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1461c99ce1acccd0c930b472c9a7c1: 2024-11-17T22:41:13,578 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-17T22:41:13,579 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:13,579 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:13,579 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:13,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3276 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3402 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3293 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3284 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3376 2024-11-17T22:41:13,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T22:41:13,579 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T22:41:13,579 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T22:41:13,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:13,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:13,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:13,580 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T22:41:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T22:41:13,583 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883273582"}]},"ts":"1731883273582"} 2024-11-17T22:41:13,584 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T22:41:13,584 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T22:41:13,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T22:41:13,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1461c99ce1acccd0c930b472c9a7c1, UNASSIGN}] 2024-11-17T22:41:13,586 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1461c99ce1acccd0c930b472c9a7c1, UNASSIGN 2024-11-17T22:41:13,586 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=ad1461c99ce1acccd0c930b472c9a7c1, regionState=CLOSING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:13,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1461c99ce1acccd0c930b472c9a7c1, UNASSIGN because future has completed 2024-11-17T22:41:13,587 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T22:41:13,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:41:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T22:41:13,740 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:13,740 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T22:41:13,740 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing ad1461c99ce1acccd0c930b472c9a7c1, disabling compactions & flushes 2024-11-17T22:41:13,740 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:13,740 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:13,740 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. after waiting 0 ms 2024-11-17T22:41:13,740 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:13,741 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(2902): Flushing ad1461c99ce1acccd0c930b472c9a7c1 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-17T22:41:13,748 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/b2fe843d14cd4a35ab44112d5f21365c is 50, key is test_row_0/A:col10/1731883273577/Put/seqid=0 2024-11-17T22:41:13,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742160_1336 (size=9857) 2024-11-17T22:41:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T22:41:14,152 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/b2fe843d14cd4a35ab44112d5f21365c 2024-11-17T22:41:14,158 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/1aa9e03857b444da9c137d492e9ed49a is 50, key is test_row_0/B:col10/1731883273577/Put/seqid=0 2024-11-17T22:41:14,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742161_1337 (size=9857) 2024-11-17T22:41:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T22:41:14,563 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/1aa9e03857b444da9c137d492e9ed49a 2024-11-17T22:41:14,576 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/66910fcbcb024642b88995f9dbf25819 is 50, key is test_row_0/C:col10/1731883273577/Put/seqid=0 2024-11-17T22:41:14,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742162_1338 (size=9857) 2024-11-17T22:41:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T22:41:14,981 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/66910fcbcb024642b88995f9dbf25819 2024-11-17T22:41:14,987 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/A/b2fe843d14cd4a35ab44112d5f21365c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/b2fe843d14cd4a35ab44112d5f21365c 2024-11-17T22:41:14,992 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/b2fe843d14cd4a35ab44112d5f21365c, entries=100, sequenceid=526, filesize=9.6 K 2024-11-17T22:41:14,993 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/B/1aa9e03857b444da9c137d492e9ed49a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/1aa9e03857b444da9c137d492e9ed49a 2024-11-17T22:41:14,999 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/1aa9e03857b444da9c137d492e9ed49a, entries=100, sequenceid=526, filesize=9.6 K 2024-11-17T22:41:15,000 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/.tmp/C/66910fcbcb024642b88995f9dbf25819 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/66910fcbcb024642b88995f9dbf25819 2024-11-17T22:41:15,004 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/66910fcbcb024642b88995f9dbf25819, entries=100, sequenceid=526, filesize=9.6 K 2024-11-17T22:41:15,005 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for ad1461c99ce1acccd0c930b472c9a7c1 in 1265ms, sequenceid=526, compaction requested=true 2024-11-17T22:41:15,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/e4cf5bbfdc234ca99572c427b970940c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ce47feb310354e9a8a616809e98d075f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/8af6d94466aa4e9f9dda01258a2ce1ef, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d3831fcdd5734ddf9b5b8cd3e8201f28, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ffcaf29af53a4c499ebd3a73f8b4cfc7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/49cc2c9e167a43abae9cc8a12a9d4d7e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/3a7f904b78b0498580545f8c338c74b9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/daad925cd59f4cdfb2a0eb4d10ba5f05, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/51b8604e80e8479181b83fb6cfdbe56a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ad7fa1f9dfb34519b11ef83cc40d8223, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/7ad6caf683914140a1c2c7bb8d2d7f95, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/18bab74f5331464681dd2a943af1cd4e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d4db0d4b2dc145d89525a4772c0ada07, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ee5e197e255643f6b690f79cb0e0fb2b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ddafb3ba44954287a3b08f8b43e332d7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/23721b5265fa4ab7b2b3392d17c24b83, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/6323d7ed618b4947954ae5f01b896d29, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/29bf0173d85a49bab9b4f3b8d133bbbc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/dc83d7008f1649d3bbea1d14b719cd3b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/4fa56db47ee249adb14e91004d886d5e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/debc46a123d543c99f0db99e63b151d7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/f18062bdf035483d8362ca80fae4f81c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d57409bd30f54b959f0a71ad8c2c4731, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/03662fa777b34685bedfccd9aa45c76f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/b61c1ca958cc469999789342c488d4ed, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/33c10a2f4b6f4748a4e86419ac3d06d6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/592eec25accf4244aea0188b089e092c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/39b81df1652a48ffaa38458bee7de2ca, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/a09205edb3f749e78149f9ce1bb830d4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/c51c455fdf414cd6959f531602869daf, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/04386a80f3f24d4281b2a8a71a66fbb1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/8eb81969037e4f1f91e813a2cc990a50, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/06a1ec87e9034be292bd718f4a75eb3d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/a2a9dfc1d79b428ebc3975961845abf9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/dc37d09a44d84048adc90d9be7f74f8e] to archive 2024-11-17T22:41:15,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:41:15,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/e4cf5bbfdc234ca99572c427b970940c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/e4cf5bbfdc234ca99572c427b970940c 2024-11-17T22:41:15,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ce47feb310354e9a8a616809e98d075f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ce47feb310354e9a8a616809e98d075f 2024-11-17T22:41:15,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/8af6d94466aa4e9f9dda01258a2ce1ef to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/8af6d94466aa4e9f9dda01258a2ce1ef 2024-11-17T22:41:15,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d3831fcdd5734ddf9b5b8cd3e8201f28 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d3831fcdd5734ddf9b5b8cd3e8201f28 2024-11-17T22:41:15,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ffcaf29af53a4c499ebd3a73f8b4cfc7 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ffcaf29af53a4c499ebd3a73f8b4cfc7 2024-11-17T22:41:15,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/49cc2c9e167a43abae9cc8a12a9d4d7e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/49cc2c9e167a43abae9cc8a12a9d4d7e 2024-11-17T22:41:15,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/3a7f904b78b0498580545f8c338c74b9 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/3a7f904b78b0498580545f8c338c74b9 2024-11-17T22:41:15,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/daad925cd59f4cdfb2a0eb4d10ba5f05 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/daad925cd59f4cdfb2a0eb4d10ba5f05 2024-11-17T22:41:15,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/51b8604e80e8479181b83fb6cfdbe56a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/51b8604e80e8479181b83fb6cfdbe56a 2024-11-17T22:41:15,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ad7fa1f9dfb34519b11ef83cc40d8223 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ad7fa1f9dfb34519b11ef83cc40d8223 2024-11-17T22:41:15,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/7ad6caf683914140a1c2c7bb8d2d7f95 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/7ad6caf683914140a1c2c7bb8d2d7f95 2024-11-17T22:41:15,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/18bab74f5331464681dd2a943af1cd4e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/18bab74f5331464681dd2a943af1cd4e 2024-11-17T22:41:15,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d4db0d4b2dc145d89525a4772c0ada07 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d4db0d4b2dc145d89525a4772c0ada07 2024-11-17T22:41:15,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ee5e197e255643f6b690f79cb0e0fb2b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ee5e197e255643f6b690f79cb0e0fb2b 2024-11-17T22:41:15,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ddafb3ba44954287a3b08f8b43e332d7 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/ddafb3ba44954287a3b08f8b43e332d7 2024-11-17T22:41:15,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/23721b5265fa4ab7b2b3392d17c24b83 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/23721b5265fa4ab7b2b3392d17c24b83 2024-11-17T22:41:15,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/6323d7ed618b4947954ae5f01b896d29 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/6323d7ed618b4947954ae5f01b896d29 2024-11-17T22:41:15,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/29bf0173d85a49bab9b4f3b8d133bbbc to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/29bf0173d85a49bab9b4f3b8d133bbbc 2024-11-17T22:41:15,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/dc83d7008f1649d3bbea1d14b719cd3b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/dc83d7008f1649d3bbea1d14b719cd3b 2024-11-17T22:41:15,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/4fa56db47ee249adb14e91004d886d5e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/4fa56db47ee249adb14e91004d886d5e 2024-11-17T22:41:15,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/debc46a123d543c99f0db99e63b151d7 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/debc46a123d543c99f0db99e63b151d7 2024-11-17T22:41:15,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/f18062bdf035483d8362ca80fae4f81c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/f18062bdf035483d8362ca80fae4f81c 2024-11-17T22:41:15,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d57409bd30f54b959f0a71ad8c2c4731 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d57409bd30f54b959f0a71ad8c2c4731 2024-11-17T22:41:15,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/03662fa777b34685bedfccd9aa45c76f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/03662fa777b34685bedfccd9aa45c76f 2024-11-17T22:41:15,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/b61c1ca958cc469999789342c488d4ed to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/b61c1ca958cc469999789342c488d4ed 2024-11-17T22:41:15,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/33c10a2f4b6f4748a4e86419ac3d06d6 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/33c10a2f4b6f4748a4e86419ac3d06d6 2024-11-17T22:41:15,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/592eec25accf4244aea0188b089e092c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/592eec25accf4244aea0188b089e092c 2024-11-17T22:41:15,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/39b81df1652a48ffaa38458bee7de2ca to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/39b81df1652a48ffaa38458bee7de2ca 2024-11-17T22:41:15,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/a09205edb3f749e78149f9ce1bb830d4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/a09205edb3f749e78149f9ce1bb830d4 2024-11-17T22:41:15,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/c51c455fdf414cd6959f531602869daf to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/c51c455fdf414cd6959f531602869daf 2024-11-17T22:41:15,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/04386a80f3f24d4281b2a8a71a66fbb1 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/04386a80f3f24d4281b2a8a71a66fbb1 2024-11-17T22:41:15,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/8eb81969037e4f1f91e813a2cc990a50 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/8eb81969037e4f1f91e813a2cc990a50 2024-11-17T22:41:15,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/06a1ec87e9034be292bd718f4a75eb3d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/06a1ec87e9034be292bd718f4a75eb3d 2024-11-17T22:41:15,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/a2a9dfc1d79b428ebc3975961845abf9 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/a2a9dfc1d79b428ebc3975961845abf9 2024-11-17T22:41:15,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/dc37d09a44d84048adc90d9be7f74f8e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/dc37d09a44d84048adc90d9be7f74f8e 2024-11-17T22:41:15,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5dceec0d506e46a88d7823468d147327, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/1715ffabebb449239214867eab71ab9c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/3523d91eb864401ca35e22a3ed46de25, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/139a95b0b11c4beb8ead5aa5d7a3a0d8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b2f75a89d5dc492bb1133c3a5e7ae8f7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2dfd6957290e42b8ae15a0eaefec807e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/6cde69906bb4448191abafee40f5bd2f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4c4c03ae5941411aa6091193aebb7b9b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e022dbc759c04fb8b171d5b1bfe21e1d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/d25964c98cc747b8bff060f579a5c592, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/02ca8c1fd3f74f1baa92e734eb5f1c0e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/483b6060fa4e417789874fa7b08a827b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/3b7c11ecebba4ae7920b173707918b57, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b287de9a44be491492301bbfe0b55662, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b38b9fa652a7429ab3995c1348be438c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b20a894fd1284115a9590d3fe31d7f12, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/021bb82f0ca14e428af278a849e9c804, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2c7e6c49d19a4bf88ffb8b40ae4d664e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/20cd98b371784a1ea90b564b3c0ec173, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/553adb3aac7a43c499ee38620c72219d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/805782a0bfc44fc88a080d4edab8033e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/92dcb3fcd66f48a6b100248db8f1dba1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/9c6748a662fe4910a9eb46cb2f20e5aa, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e26a835c93d04774a128db98d7b07e33, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e04810c6e48e4a9aafe1f7c6b37d2f82, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/f71e2d5e9bde47cfbda7a4b024fde94a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4e83d6fda2404dee95ad249825427642, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/aeb3b972bc6942b89279d4941d69e9c0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b3968eec34764eefb1b001e69eb308c4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/7713ecc34cab4cc699d484238aac4024, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/a46b15504772477dab45ba1778ae588c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5599248bfba14876ab599994363c23fb, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/19c2c51d3f6a45f9a6a5e3a21dfab478, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/0c35b70ee2a9418fb22b586c4dc5b97c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/857c3d635431439c8120d05a72a639a6] to archive 2024-11-17T22:41:15,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:41:15,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5dceec0d506e46a88d7823468d147327 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5dceec0d506e46a88d7823468d147327 2024-11-17T22:41:15,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/1715ffabebb449239214867eab71ab9c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/1715ffabebb449239214867eab71ab9c 2024-11-17T22:41:15,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/3523d91eb864401ca35e22a3ed46de25 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/3523d91eb864401ca35e22a3ed46de25 2024-11-17T22:41:15,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/139a95b0b11c4beb8ead5aa5d7a3a0d8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/139a95b0b11c4beb8ead5aa5d7a3a0d8 2024-11-17T22:41:15,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b2f75a89d5dc492bb1133c3a5e7ae8f7 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b2f75a89d5dc492bb1133c3a5e7ae8f7 2024-11-17T22:41:15,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2dfd6957290e42b8ae15a0eaefec807e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2dfd6957290e42b8ae15a0eaefec807e 2024-11-17T22:41:15,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/6cde69906bb4448191abafee40f5bd2f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/6cde69906bb4448191abafee40f5bd2f 2024-11-17T22:41:15,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4c4c03ae5941411aa6091193aebb7b9b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4c4c03ae5941411aa6091193aebb7b9b 2024-11-17T22:41:15,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e022dbc759c04fb8b171d5b1bfe21e1d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e022dbc759c04fb8b171d5b1bfe21e1d 2024-11-17T22:41:15,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/d25964c98cc747b8bff060f579a5c592 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/d25964c98cc747b8bff060f579a5c592 2024-11-17T22:41:15,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/02ca8c1fd3f74f1baa92e734eb5f1c0e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/02ca8c1fd3f74f1baa92e734eb5f1c0e 2024-11-17T22:41:15,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/483b6060fa4e417789874fa7b08a827b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/483b6060fa4e417789874fa7b08a827b 2024-11-17T22:41:15,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/3b7c11ecebba4ae7920b173707918b57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/3b7c11ecebba4ae7920b173707918b57 2024-11-17T22:41:15,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b287de9a44be491492301bbfe0b55662 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b287de9a44be491492301bbfe0b55662 2024-11-17T22:41:15,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b38b9fa652a7429ab3995c1348be438c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b38b9fa652a7429ab3995c1348be438c 2024-11-17T22:41:15,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b20a894fd1284115a9590d3fe31d7f12 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b20a894fd1284115a9590d3fe31d7f12 2024-11-17T22:41:15,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/021bb82f0ca14e428af278a849e9c804 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/021bb82f0ca14e428af278a849e9c804 2024-11-17T22:41:15,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2c7e6c49d19a4bf88ffb8b40ae4d664e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2c7e6c49d19a4bf88ffb8b40ae4d664e 2024-11-17T22:41:15,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/20cd98b371784a1ea90b564b3c0ec173 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/20cd98b371784a1ea90b564b3c0ec173 2024-11-17T22:41:15,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/553adb3aac7a43c499ee38620c72219d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/553adb3aac7a43c499ee38620c72219d 2024-11-17T22:41:15,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/805782a0bfc44fc88a080d4edab8033e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/805782a0bfc44fc88a080d4edab8033e 2024-11-17T22:41:15,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/92dcb3fcd66f48a6b100248db8f1dba1 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/92dcb3fcd66f48a6b100248db8f1dba1 2024-11-17T22:41:15,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/9c6748a662fe4910a9eb46cb2f20e5aa to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/9c6748a662fe4910a9eb46cb2f20e5aa 2024-11-17T22:41:15,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e26a835c93d04774a128db98d7b07e33 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e26a835c93d04774a128db98d7b07e33 2024-11-17T22:41:15,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e04810c6e48e4a9aafe1f7c6b37d2f82 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/e04810c6e48e4a9aafe1f7c6b37d2f82 2024-11-17T22:41:15,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/f71e2d5e9bde47cfbda7a4b024fde94a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/f71e2d5e9bde47cfbda7a4b024fde94a 2024-11-17T22:41:15,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4e83d6fda2404dee95ad249825427642 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/4e83d6fda2404dee95ad249825427642 2024-11-17T22:41:15,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/aeb3b972bc6942b89279d4941d69e9c0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/aeb3b972bc6942b89279d4941d69e9c0 2024-11-17T22:41:15,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b3968eec34764eefb1b001e69eb308c4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/b3968eec34764eefb1b001e69eb308c4 2024-11-17T22:41:15,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/7713ecc34cab4cc699d484238aac4024 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/7713ecc34cab4cc699d484238aac4024 2024-11-17T22:41:15,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/a46b15504772477dab45ba1778ae588c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/a46b15504772477dab45ba1778ae588c 2024-11-17T22:41:15,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5599248bfba14876ab599994363c23fb to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/5599248bfba14876ab599994363c23fb 2024-11-17T22:41:15,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/19c2c51d3f6a45f9a6a5e3a21dfab478 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/19c2c51d3f6a45f9a6a5e3a21dfab478 2024-11-17T22:41:15,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/0c35b70ee2a9418fb22b586c4dc5b97c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/0c35b70ee2a9418fb22b586c4dc5b97c 2024-11-17T22:41:15,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/857c3d635431439c8120d05a72a639a6 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/857c3d635431439c8120d05a72a639a6 2024-11-17T22:41:15,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7830e5e49a424a009d0142f94e8cb940, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7917c69aae16441da70ee5561f0557f2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/9ec0334c42f446b6a4338f391af11aaf, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/16a14ee51d074a70891e85f91b6fed5c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2053ef2925bb4b5dadd5ed76f01b0f6b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/ce7ea50943214167a3508be4efc159d2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3da659b8923a4e52a2322453a2abf8e1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/fa1b1445d2c544ccb2bd6129d68c2c52, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7598b3b647d7440788d9db93e095c4f6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0586271406c942c7b5912ceefb3c3741, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/1757848d5bc64827a44dd2237d1fb97b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0b7c7b67e9bf4b28b72c4651ae530e25, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4ad0ee705d804f1588fd4370f1d68164, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/ebbb9f0cd573434fa49c75059b1a7922, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2d62f133ebfb4a1d99bb20e14e807dba, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4794f5b3317141d7bdc8db0e07fde8b8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/71517a39671944af8bcf11a3a569d48d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f3f210a347e490dbab0953554d06021, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4bc87ef0be144160a1d4c23cfc1fe3f4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/53c556d7b70e490c9f1ec51be6b3e6d9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/a3d49aea18cb4598b1b43da62a84acaf, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f9419e8552b344a69df488789bd50217, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/38ab749d024f41ffb8897d14c2219f91, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/9cc8bdddf09e4d3a8f9ed0597aa619f4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/03b2a81d773242588f4b2bc3bf0ad7e8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/348016686e034ca3a76f97af90089705, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3391b1cb877b474d89ceaf351277a925, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f46feba8224a47318d8a0fcaa8d13e4b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/57fda99b772849d1b35e7844ae24c443, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/139256c443e7400cbeb4efae21412166, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/241aa18ea2a54419bd62a4591498be92, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f565bb4aba74377affc26d8b2d781e1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0a5e6c6d10264c478f6a2b0f0c8cc46a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/628651909d4341e4bc0d86121485382d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f6dab66b3450442180f3b67f8a668805] to archive 2024-11-17T22:41:15,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:41:15,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7830e5e49a424a009d0142f94e8cb940 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7830e5e49a424a009d0142f94e8cb940 2024-11-17T22:41:15,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7917c69aae16441da70ee5561f0557f2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7917c69aae16441da70ee5561f0557f2 2024-11-17T22:41:15,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/9ec0334c42f446b6a4338f391af11aaf to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/9ec0334c42f446b6a4338f391af11aaf 2024-11-17T22:41:15,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/16a14ee51d074a70891e85f91b6fed5c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/16a14ee51d074a70891e85f91b6fed5c 2024-11-17T22:41:15,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2053ef2925bb4b5dadd5ed76f01b0f6b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2053ef2925bb4b5dadd5ed76f01b0f6b 2024-11-17T22:41:15,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/ce7ea50943214167a3508be4efc159d2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/ce7ea50943214167a3508be4efc159d2 2024-11-17T22:41:15,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3da659b8923a4e52a2322453a2abf8e1 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3da659b8923a4e52a2322453a2abf8e1 2024-11-17T22:41:15,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/fa1b1445d2c544ccb2bd6129d68c2c52 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/fa1b1445d2c544ccb2bd6129d68c2c52 2024-11-17T22:41:15,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7598b3b647d7440788d9db93e095c4f6 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/7598b3b647d7440788d9db93e095c4f6 2024-11-17T22:41:15,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0586271406c942c7b5912ceefb3c3741 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0586271406c942c7b5912ceefb3c3741 2024-11-17T22:41:15,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/1757848d5bc64827a44dd2237d1fb97b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/1757848d5bc64827a44dd2237d1fb97b 2024-11-17T22:41:15,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0b7c7b67e9bf4b28b72c4651ae530e25 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0b7c7b67e9bf4b28b72c4651ae530e25 2024-11-17T22:41:15,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4ad0ee705d804f1588fd4370f1d68164 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4ad0ee705d804f1588fd4370f1d68164 2024-11-17T22:41:15,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/ebbb9f0cd573434fa49c75059b1a7922 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/ebbb9f0cd573434fa49c75059b1a7922 2024-11-17T22:41:15,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2d62f133ebfb4a1d99bb20e14e807dba to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/2d62f133ebfb4a1d99bb20e14e807dba 2024-11-17T22:41:15,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4794f5b3317141d7bdc8db0e07fde8b8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4794f5b3317141d7bdc8db0e07fde8b8 2024-11-17T22:41:15,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/71517a39671944af8bcf11a3a569d48d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/71517a39671944af8bcf11a3a569d48d 2024-11-17T22:41:15,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f3f210a347e490dbab0953554d06021 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f3f210a347e490dbab0953554d06021 2024-11-17T22:41:15,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4bc87ef0be144160a1d4c23cfc1fe3f4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4bc87ef0be144160a1d4c23cfc1fe3f4 2024-11-17T22:41:15,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/53c556d7b70e490c9f1ec51be6b3e6d9 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/53c556d7b70e490c9f1ec51be6b3e6d9 2024-11-17T22:41:15,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/a3d49aea18cb4598b1b43da62a84acaf to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/a3d49aea18cb4598b1b43da62a84acaf 2024-11-17T22:41:15,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f9419e8552b344a69df488789bd50217 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f9419e8552b344a69df488789bd50217 2024-11-17T22:41:15,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/38ab749d024f41ffb8897d14c2219f91 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/38ab749d024f41ffb8897d14c2219f91 2024-11-17T22:41:15,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/9cc8bdddf09e4d3a8f9ed0597aa619f4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/9cc8bdddf09e4d3a8f9ed0597aa619f4 2024-11-17T22:41:15,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/03b2a81d773242588f4b2bc3bf0ad7e8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/03b2a81d773242588f4b2bc3bf0ad7e8 2024-11-17T22:41:15,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/348016686e034ca3a76f97af90089705 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/348016686e034ca3a76f97af90089705 2024-11-17T22:41:15,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3391b1cb877b474d89ceaf351277a925 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/3391b1cb877b474d89ceaf351277a925 2024-11-17T22:41:15,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f46feba8224a47318d8a0fcaa8d13e4b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f46feba8224a47318d8a0fcaa8d13e4b 2024-11-17T22:41:15,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/57fda99b772849d1b35e7844ae24c443 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/57fda99b772849d1b35e7844ae24c443 2024-11-17T22:41:15,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/139256c443e7400cbeb4efae21412166 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/139256c443e7400cbeb4efae21412166 2024-11-17T22:41:15,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/241aa18ea2a54419bd62a4591498be92 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/241aa18ea2a54419bd62a4591498be92 2024-11-17T22:41:15,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f565bb4aba74377affc26d8b2d781e1 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/4f565bb4aba74377affc26d8b2d781e1 2024-11-17T22:41:15,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0a5e6c6d10264c478f6a2b0f0c8cc46a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/0a5e6c6d10264c478f6a2b0f0c8cc46a 2024-11-17T22:41:15,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/628651909d4341e4bc0d86121485382d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/628651909d4341e4bc0d86121485382d 2024-11-17T22:41:15,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f6dab66b3450442180f3b67f8a668805 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/f6dab66b3450442180f3b67f8a668805 2024-11-17T22:41:15,108 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/recovered.edits/529.seqid, newMaxSeqId=529, maxSeqId=1 2024-11-17T22:41:15,108 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. 2024-11-17T22:41:15,109 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for ad1461c99ce1acccd0c930b472c9a7c1: Waiting for close lock at 1731883273740Running coprocessor pre-close hooks at 1731883273740Disabling compacts and flushes for region at 1731883273740Disabling writes for close at 1731883273740Obtaining lock to block concurrent updates at 1731883273741 (+1 ms)Preparing flush snapshotting stores in ad1461c99ce1acccd0c930b472c9a7c1 at 1731883273741Finished memstore snapshotting TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731883273741Flushing stores of TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1. at 1731883273742 (+1 ms)Flushing ad1461c99ce1acccd0c930b472c9a7c1/A: creating writer at 1731883273742Flushing ad1461c99ce1acccd0c930b472c9a7c1/A: appending metadata at 1731883273747 (+5 ms)Flushing ad1461c99ce1acccd0c930b472c9a7c1/A: closing flushed file at 1731883273747Flushing ad1461c99ce1acccd0c930b472c9a7c1/B: creating writer at 1731883274156 (+409 ms)Flushing ad1461c99ce1acccd0c930b472c9a7c1/B: appending metadata at 1731883274158 (+2 ms)Flushing ad1461c99ce1acccd0c930b472c9a7c1/B: closing flushed file at 1731883274158Flushing ad1461c99ce1acccd0c930b472c9a7c1/C: creating writer at 1731883274571 (+413 ms)Flushing ad1461c99ce1acccd0c930b472c9a7c1/C: appending metadata at 1731883274575 (+4 ms)Flushing ad1461c99ce1acccd0c930b472c9a7c1/C: closing flushed file at 1731883274575Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7239bf6d: reopening flushed file at 1731883274986 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@483deeee: reopening flushed file at 1731883274992 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@531b070c: reopening flushed file at 1731883274999 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for ad1461c99ce1acccd0c930b472c9a7c1 in 1265ms, sequenceid=526, compaction requested=true at 1731883275005 (+6 ms)Writing region close event to WAL at 1731883275106 (+101 ms)Running coprocessor post-close hooks at 1731883275108 (+2 ms)Closed at 1731883275108 2024-11-17T22:41:15,110 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:15,111 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=ad1461c99ce1acccd0c930b472c9a7c1, regionState=CLOSED 2024-11-17T22:41:15,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:41:15,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-17T22:41:15,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure ad1461c99ce1acccd0c930b472c9a7c1, server=1a6e40b21a48,43011,1731883188895 in 1.5260 sec 2024-11-17T22:41:15,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-17T22:41:15,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1461c99ce1acccd0c930b472c9a7c1, UNASSIGN in 1.5290 sec 2024-11-17T22:41:15,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-17T22:41:15,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5320 sec 2024-11-17T22:41:15,118 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883275118"}]},"ts":"1731883275118"} 2024-11-17T22:41:15,119 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T22:41:15,119 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T22:41:15,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5400 sec 2024-11-17T22:41:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T22:41:15,728 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:15,729 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T22:41:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:15,733 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=100, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-17T22:41:15,735 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=100, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:15,740 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:15,744 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/recovered.edits] 2024-11-17T22:41:15,747 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/3eac31fb00b44b51b35cac219c9899c3 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/3eac31fb00b44b51b35cac219c9899c3 2024-11-17T22:41:15,749 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/b2fe843d14cd4a35ab44112d5f21365c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/b2fe843d14cd4a35ab44112d5f21365c 2024-11-17T22:41:15,750 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d77dbf289ae64693bd8e35dc1c8e5d65 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/A/d77dbf289ae64693bd8e35dc1c8e5d65 2024-11-17T22:41:15,753 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/1aa9e03857b444da9c137d492e9ed49a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/1aa9e03857b444da9c137d492e9ed49a 2024-11-17T22:41:15,754 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2b3612ee1d384bc880b32687f1167d5c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/2b3612ee1d384bc880b32687f1167d5c 2024-11-17T22:41:15,755 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/ef5632c3821e42c09b3f6d815ad097c4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/B/ef5632c3821e42c09b3f6d815ad097c4 2024-11-17T22:41:15,758 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/66910fcbcb024642b88995f9dbf25819 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/66910fcbcb024642b88995f9dbf25819 2024-11-17T22:41:15,759 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/6976074313e340e0875d1077cae1e572 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/6976074313e340e0875d1077cae1e572 2024-11-17T22:41:15,761 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/d5722a469d224edbbc7215f58ad26e71 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/C/d5722a469d224edbbc7215f58ad26e71 2024-11-17T22:41:15,764 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/recovered.edits/529.seqid to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1/recovered.edits/529.seqid 2024-11-17T22:41:15,765 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/ad1461c99ce1acccd0c930b472c9a7c1 2024-11-17T22:41:15,765 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T22:41:15,768 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=100, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:15,773 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T22:41:15,776 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T22:41:15,777 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=100, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:15,777 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T22:41:15,777 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731883275777"}]},"ts":"9223372036854775807"} 2024-11-17T22:41:15,779 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T22:41:15,779 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ad1461c99ce1acccd0c930b472c9a7c1, NAME => 'TestAcidGuarantees,,1731883248154.ad1461c99ce1acccd0c930b472c9a7c1.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T22:41:15,779 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T22:41:15,779 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731883275779"}]},"ts":"9223372036854775807"} 2024-11-17T22:41:15,781 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T22:41:15,782 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=100, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:15,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-11-17T22:41:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-17T22:41:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T22:41:15,837 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:15,844 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 240), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 399) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4620 (was 4699) 2024-11-17T22:41:15,852 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=4619 2024-11-17T22:41:15,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T22:41:15,854 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T22:41:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:15,856 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T22:41:15,856 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:15,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 101 2024-11-17T22:41:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T22:41:15,856 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T22:41:15,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742163_1339 (size=959) 2024-11-17T22:41:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T22:41:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T22:41:16,266 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71 2024-11-17T22:41:16,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742164_1340 (size=53) 2024-11-17T22:41:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T22:41:16,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:41:16,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing dd1e0e7a755612e5d2f97d7c99dd3f57, disabling compactions & flushes 2024-11-17T22:41:16,676 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:16,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:16,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. after waiting 0 ms 2024-11-17T22:41:16,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:16,676 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:16,677 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for dd1e0e7a755612e5d2f97d7c99dd3f57: Waiting for close lock at 1731883276676Disabling compacts and flushes for region at 1731883276676Disabling writes for close at 1731883276676Writing region close event to WAL at 1731883276676Closed at 1731883276676 2024-11-17T22:41:16,679 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T22:41:16,680 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731883276679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731883276679"}]},"ts":"1731883276679"} 2024-11-17T22:41:16,682 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T22:41:16,683 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T22:41:16,683 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883276683"}]},"ts":"1731883276683"} 2024-11-17T22:41:16,685 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T22:41:16,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, ASSIGN}] 2024-11-17T22:41:16,686 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, ASSIGN 2024-11-17T22:41:16,687 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, ASSIGN; state=OFFLINE, location=1a6e40b21a48,43011,1731883188895; forceNewPlan=false, retain=false 2024-11-17T22:41:16,838 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=dd1e0e7a755612e5d2f97d7c99dd3f57, regionState=OPENING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:16,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, ASSIGN because future has completed 2024-11-17T22:41:16,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:41:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T22:41:17,002 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:17,002 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7752): Opening region: {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} 2024-11-17T22:41:17,002 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,002 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:41:17,003 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7794): checking encryption for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,003 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7797): checking classloading for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,004 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,005 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd1e0e7a755612e5d2f97d7c99dd3f57 columnFamilyName A 2024-11-17T22:41:17,005 DEBUG [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:17,005 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(327): Store=dd1e0e7a755612e5d2f97d7c99dd3f57/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:41:17,006 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,007 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd1e0e7a755612e5d2f97d7c99dd3f57 columnFamilyName B 2024-11-17T22:41:17,007 DEBUG [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:17,007 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(327): Store=dd1e0e7a755612e5d2f97d7c99dd3f57/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:41:17,007 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,008 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd1e0e7a755612e5d2f97d7c99dd3f57 columnFamilyName C 2024-11-17T22:41:17,008 DEBUG [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:17,008 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(327): Store=dd1e0e7a755612e5d2f97d7c99dd3f57/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:41:17,008 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1038): replaying wal for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,009 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,009 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,010 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1048): stopping wal replay for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,010 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1060): Cleaning up temporary data for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,010 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T22:41:17,011 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1093): writing seq id for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,013 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T22:41:17,013 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1114): Opened dd1e0e7a755612e5d2f97d7c99dd3f57; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65718585, jitterRate=-0.020716771483421326}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T22:41:17,013 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:17,014 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1006): Region open journal for dd1e0e7a755612e5d2f97d7c99dd3f57: Running coprocessor pre-open hook at 1731883277003Writing region info on filesystem at 1731883277003Initializing all the Stores at 1731883277003Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883277003Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883277004 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883277004Cleaning up temporary data from old regions at 1731883277010 (+6 ms)Running coprocessor post-open hooks at 1731883277013 (+3 ms)Region opened successfully at 1731883277014 (+1 ms) 2024-11-17T22:41:17,015 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., pid=103, masterSystemTime=1731883276996 2024-11-17T22:41:17,016 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:17,016 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:17,017 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=dd1e0e7a755612e5d2f97d7c99dd3f57, regionState=OPEN, openSeqNum=2, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:17,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:41:17,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-17T22:41:17,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 in 177 msec 2024-11-17T22:41:17,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-17T22:41:17,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, ASSIGN in 335 msec 2024-11-17T22:41:17,022 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T22:41:17,023 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883277022"}]},"ts":"1731883277022"} 2024-11-17T22:41:17,024 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T22:41:17,025 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T22:41:17,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-11-17T22:41:17,246 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T22:41:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T22:41:18,007 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:18,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecc882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:18,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:18,010 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b8b5c7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:18,012 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ecee9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,013 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:18,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:18,016 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T22:41:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a6e40b21a48,45671,1731883188234 2024-11-17T22:41:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f577389 2024-11-17T22:41:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T22:41:18,018 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T22:41:18,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T22:41:18,020 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T22:41:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:18,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742165_1341 (size=995) 2024-11-17T22:41:18,439 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T22:41:18,439 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T22:41:18,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T22:41:18,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, REOPEN/MOVE}] 2024-11-17T22:41:18,449 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, REOPEN/MOVE 2024-11-17T22:41:18,450 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=dd1e0e7a755612e5d2f97d7c99dd3f57, regionState=CLOSING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:18,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, REOPEN/MOVE because future has completed 2024-11-17T22:41:18,451 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T22:41:18,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE, hasLock=false; CloseRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:41:18,604 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] handler.UnassignRegionHandler(122): Close dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,604 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T22:41:18,604 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1722): Closing dd1e0e7a755612e5d2f97d7c99dd3f57, disabling compactions & flushes 2024-11-17T22:41:18,604 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:18,604 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:18,604 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. after waiting 0 ms 2024-11-17T22:41:18,604 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:18,613 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T22:41:18,614 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:18,614 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1676): Region close journal for dd1e0e7a755612e5d2f97d7c99dd3f57: Waiting for close lock at 1731883278604Running coprocessor pre-close hooks at 1731883278604Disabling compacts and flushes for region at 1731883278604Disabling writes for close at 1731883278604Writing region close event to WAL at 1731883278605 (+1 ms)Running coprocessor post-close hooks at 1731883278614 (+9 ms)Closed at 1731883278614 2024-11-17T22:41:18,615 WARN [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegionServer(3299): Not adding moved region record: dd1e0e7a755612e5d2f97d7c99dd3f57 to self. 2024-11-17T22:41:18,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T22:41:18,618 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] handler.UnassignRegionHandler(157): Closed dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,619 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=dd1e0e7a755612e5d2f97d7c99dd3f57, regionState=CLOSED 2024-11-17T22:41:18,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE, hasLock=false; CloseRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:41:18,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-17T22:41:18,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; CloseRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 in 170 msec 2024-11-17T22:41:18,624 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, REOPEN/MOVE; state=CLOSED, location=1a6e40b21a48,43011,1731883188895; forceNewPlan=false, retain=true 2024-11-17T22:41:18,775 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=dd1e0e7a755612e5d2f97d7c99dd3f57, regionState=OPENING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:18,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, REOPEN/MOVE because future has completed 2024-11-17T22:41:18,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=106, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:41:18,937 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:18,938 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} 2024-11-17T22:41:18,938 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,938 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:41:18,939 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,939 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,941 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,943 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd1e0e7a755612e5d2f97d7c99dd3f57 columnFamilyName A 2024-11-17T22:41:18,945 DEBUG [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:18,945 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(327): Store=dd1e0e7a755612e5d2f97d7c99dd3f57/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:41:18,946 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,947 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd1e0e7a755612e5d2f97d7c99dd3f57 columnFamilyName B 2024-11-17T22:41:18,947 DEBUG [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:18,947 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(327): Store=dd1e0e7a755612e5d2f97d7c99dd3f57/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:41:18,948 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,948 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd1e0e7a755612e5d2f97d7c99dd3f57 columnFamilyName C 2024-11-17T22:41:18,948 DEBUG [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:18,949 INFO [StoreOpener-dd1e0e7a755612e5d2f97d7c99dd3f57-1 {}] regionserver.HStore(327): Store=dd1e0e7a755612e5d2f97d7c99dd3f57/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:41:18,949 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,949 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,951 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,952 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,952 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,952 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T22:41:18,954 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,955 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened dd1e0e7a755612e5d2f97d7c99dd3f57; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66271010, jitterRate=-0.012484997510910034}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T22:41:18,955 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:18,956 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for dd1e0e7a755612e5d2f97d7c99dd3f57: Running coprocessor pre-open hook at 1731883278939Writing region info on filesystem at 1731883278939Initializing all the Stores at 1731883278940 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883278941 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883278941Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883278941Cleaning up temporary data from old regions at 1731883278952 (+11 ms)Running coprocessor post-open hooks at 1731883278955 (+3 ms)Region opened successfully at 1731883278955 2024-11-17T22:41:18,956 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., pid=108, masterSystemTime=1731883278933 2024-11-17T22:41:18,958 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:18,958 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:18,959 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=dd1e0e7a755612e5d2f97d7c99dd3f57, regionState=OPEN, openSeqNum=5, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:18,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=106, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:41:18,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=106 2024-11-17T22:41:18,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=106, state=SUCCESS, hasLock=false; OpenRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 in 182 msec 2024-11-17T22:41:18,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-17T22:41:18,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, REOPEN/MOVE in 515 msec 2024-11-17T22:41:18,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-17T22:41:18,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-11-17T22:41:18,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 950 msec 2024-11-17T22:41:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-17T22:41:18,973 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:18,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f86bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:18,975 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:18,975 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395a98e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:18,977 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41aa56ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:18,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:18,980 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@487e0461, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:18,982 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@241aaeb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:18,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:18,985 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc0b24f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:18,987 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6649ae7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:18,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:18,990 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e27f213, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:18,992 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@142cdc7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:18,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:18,994 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b0aec45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:18,995 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e4f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:18,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:18,997 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:18,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:18,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:18,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce90a0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:18,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:18,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:18,998 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cdd5871, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:18,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:18,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:19,000 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da8c649, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:19,001 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:19,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46039ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:19,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:19,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:19,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:19,002 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56817686, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:19,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:19,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68038b02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:19,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:19,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:19,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:19,005 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213b285, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:19,007 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e6730f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:19,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:19,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:19,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:19,008 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d7564d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:19,010 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@801e9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:19,011 DEBUG [Thread-1564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:19,011 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:19,011 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:19,011 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:19,011 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:19,012 DEBUG [Thread-1547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:19,012 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:19,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:19,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:19,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:19,013 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:19,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:19,013 DEBUG [Thread-1545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:19,013 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:19,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-17T22:41:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:19,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:19,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:19,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:41:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T22:41:19,015 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:41:19,017 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:19,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:41:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:41:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:41:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:41:19,018 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:41:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:41:19,020 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:19,020 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:19,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:41:19,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:41:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:19,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:41:19,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c9fc37f645ca4e4c881ee02b63cd68f5_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883279026/Put/seqid=0 2024-11-17T22:41:19,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742166_1342 (size=12154) 2024-11-17T22:41:19,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883339063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883339064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883339069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883339071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883339072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T22:41:19,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883339177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883339177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883339186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883339186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883339186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:19,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:19,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T22:41:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883339387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883339387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883339396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883339397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883339397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,471 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:19,475 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c9fc37f645ca4e4c881ee02b63cd68f5_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c9fc37f645ca4e4c881ee02b63cd68f5_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:19,475 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/ee5353d54fe240be978b9df926e20813, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:19,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/ee5353d54fe240be978b9df926e20813 is 175, key is test_row_0/A:col10/1731883279026/Put/seqid=0 2024-11-17T22:41:19,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:19,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:19,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742167_1343 (size=30955) 2024-11-17T22:41:19,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T22:41:19,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883339697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883339697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883339706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883339707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883339707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:19,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,880 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/ee5353d54fe240be978b9df926e20813 2024-11-17T22:41:19,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/d7781f43beb24bbe8510ad840efe3fc3 is 50, key is test_row_0/B:col10/1731883279026/Put/seqid=0 2024-11-17T22:41:19,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742168_1344 (size=12001) 2024-11-17T22:41:19,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:19,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:19,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:19,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:20,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:20,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:20,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:20,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T22:41:20,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883340206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883340207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:20,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883340217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:20,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:20,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:20,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:20,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883340217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:20,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883340217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:20,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:20,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:20,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:20,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:20,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/d7781f43beb24bbe8510ad840efe3fc3 2024-11-17T22:41:20,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/a28314c892664526ba982cd6703dde3f is 50, key is test_row_0/C:col10/1731883279026/Put/seqid=0 2024-11-17T22:41:20,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742169_1345 (size=12001) 2024-11-17T22:41:20,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:20,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:20,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:20,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:20,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:20,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:20,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:20,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:20,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,637 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T22:41:20,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:20,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:20,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:20,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:20,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:20,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/a28314c892664526ba982cd6703dde3f 2024-11-17T22:41:20,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/ee5353d54fe240be978b9df926e20813 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/ee5353d54fe240be978b9df926e20813 2024-11-17T22:41:20,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/ee5353d54fe240be978b9df926e20813, entries=150, sequenceid=16, filesize=30.2 K 2024-11-17T22:41:20,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/d7781f43beb24bbe8510ad840efe3fc3 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/d7781f43beb24bbe8510ad840efe3fc3 2024-11-17T22:41:20,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/d7781f43beb24bbe8510ad840efe3fc3, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T22:41:20,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/a28314c892664526ba982cd6703dde3f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/a28314c892664526ba982cd6703dde3f 2024-11-17T22:41:20,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/a28314c892664526ba982cd6703dde3f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T22:41:20,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1742ms, sequenceid=16, compaction requested=false 2024-11-17T22:41:20,770 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T22:41:20,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:20,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T22:41:20,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:20,852 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:41:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ce6488eb7c32463f8ef9bbb375825341_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883279065/Put/seqid=0 2024-11-17T22:41:20,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742170_1346 (size=12154) 2024-11-17T22:41:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T22:41:21,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:21,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883341232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883341233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883341235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883341236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883341236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,244 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:21,270 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ce6488eb7c32463f8ef9bbb375825341_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ce6488eb7c32463f8ef9bbb375825341_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:21,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/e917cdec9e574505be1ae86ec3a4f00c, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:21,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/e917cdec9e574505be1ae86ec3a4f00c is 175, key is test_row_0/A:col10/1731883279065/Put/seqid=0 2024-11-17T22:41:21,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742171_1347 (size=30955) 2024-11-17T22:41:21,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883341346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883341346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883341346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883341347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883341557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883341557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883341557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883341557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,677 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/e917cdec9e574505be1ae86ec3a4f00c 2024-11-17T22:41:21,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/dcafcb6095024c438ead2f8ea9e98de5 is 50, key is test_row_0/B:col10/1731883279065/Put/seqid=0 2024-11-17T22:41:21,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742172_1348 (size=12001) 2024-11-17T22:41:21,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883341867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883341867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883341877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883341877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:22,088 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/dcafcb6095024c438ead2f8ea9e98de5 2024-11-17T22:41:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/c97e86ba38454c0b8725d2578eca0de9 is 50, key is test_row_0/C:col10/1731883279065/Put/seqid=0 2024-11-17T22:41:22,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742173_1349 (size=12001) 2024-11-17T22:41:22,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883342377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:22,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883342377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:22,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883342396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:22,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:22,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:22,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:22,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883342396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:22,502 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/c97e86ba38454c0b8725d2578eca0de9 2024-11-17T22:41:22,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/e917cdec9e574505be1ae86ec3a4f00c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/e917cdec9e574505be1ae86ec3a4f00c 2024-11-17T22:41:22,509 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/e917cdec9e574505be1ae86ec3a4f00c, entries=150, sequenceid=40, filesize=30.2 K 2024-11-17T22:41:22,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/dcafcb6095024c438ead2f8ea9e98de5 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/dcafcb6095024c438ead2f8ea9e98de5 2024-11-17T22:41:22,513 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/dcafcb6095024c438ead2f8ea9e98de5, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T22:41:22,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/c97e86ba38454c0b8725d2578eca0de9 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/c97e86ba38454c0b8725d2578eca0de9 2024-11-17T22:41:22,518 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/c97e86ba38454c0b8725d2578eca0de9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T22:41:22,519 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1668ms, sequenceid=40, compaction requested=false 2024-11-17T22:41:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-17T22:41:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-17T22:41:22,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-17T22:41:22,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5030 sec 2024-11-17T22:41:22,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.5090 sec 2024-11-17T22:41:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T22:41:23,168 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:23,168 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-17T22:41:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T22:41:23,169 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:23,170 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:23,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:23,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:41:23,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171fa5ece54d3f48a9a8bab560a257b616_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883281235/Put/seqid=0 2024-11-17T22:41:23,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742174_1350 (size=14594) 2024-11-17T22:41:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T22:41:23,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:23,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:23,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883343383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:23,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883343387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:23,394 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:23,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883343396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:23,404 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:23,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883343417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:23,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883343417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:23,423 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:23,423 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:23,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:23,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:23,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T22:41:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883343487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:23,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:23,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:23,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:23,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:23,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:23,677 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171fa5ece54d3f48a9a8bab560a257b616_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171fa5ece54d3f48a9a8bab560a257b616_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:23,678 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/60efd75c377b4d7ba483614a13050ea6, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:23,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/60efd75c377b4d7ba483614a13050ea6 is 175, key is test_row_0/A:col10/1731883281235/Put/seqid=0 2024-11-17T22:41:23,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742175_1351 (size=39549) 2024-11-17T22:41:23,684 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/60efd75c377b4d7ba483614a13050ea6 2024-11-17T22:41:23,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/92ee062b1ac94be0b046a008f8f2d2f5 is 50, key is test_row_0/B:col10/1731883281235/Put/seqid=0 2024-11-17T22:41:23,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742176_1352 (size=12001) 2024-11-17T22:41:23,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883343697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:23,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:23,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:23,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:23,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:23,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T22:41:23,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:23,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:23,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:23,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:23,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883344007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:24,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:24,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:24,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/92ee062b1ac94be0b046a008f8f2d2f5 2024-11-17T22:41:24,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/4e20bd70c8134eb4b4417cfd2b319fbf is 50, key is test_row_0/C:col10/1731883281235/Put/seqid=0 2024-11-17T22:41:24,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742177_1353 (size=12001) 2024-11-17T22:41:24,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T22:41:24,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:24,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/4e20bd70c8134eb4b4417cfd2b319fbf 2024-11-17T22:41:24,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/60efd75c377b4d7ba483614a13050ea6 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60efd75c377b4d7ba483614a13050ea6 2024-11-17T22:41:24,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60efd75c377b4d7ba483614a13050ea6, entries=200, sequenceid=53, filesize=38.6 K 2024-11-17T22:41:24,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/92ee062b1ac94be0b046a008f8f2d2f5 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/92ee062b1ac94be0b046a008f8f2d2f5 2024-11-17T22:41:24,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/92ee062b1ac94be0b046a008f8f2d2f5, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T22:41:24,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/4e20bd70c8134eb4b4417cfd2b319fbf as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/4e20bd70c8134eb4b4417cfd2b319fbf 2024-11-17T22:41:24,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/4e20bd70c8134eb4b4417cfd2b319fbf, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T22:41:24,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1270ms, sequenceid=53, compaction requested=true 2024-11-17T22:41:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:24,533 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:24,534 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:24,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:24,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:24,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:24,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:24,535 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:24,535 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:24,535 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/B is initiating minor compaction (all files) 2024-11-17T22:41:24,535 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/A is initiating minor compaction (all files) 2024-11-17T22:41:24,535 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/A in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,535 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/B in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,535 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/d7781f43beb24bbe8510ad840efe3fc3, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/dcafcb6095024c438ead2f8ea9e98de5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/92ee062b1ac94be0b046a008f8f2d2f5] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=35.2 K 2024-11-17T22:41:24,535 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/ee5353d54fe240be978b9df926e20813, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/e917cdec9e574505be1ae86ec3a4f00c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60efd75c377b4d7ba483614a13050ea6] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=99.1 K 2024-11-17T22:41:24,535 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,535 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/ee5353d54fe240be978b9df926e20813, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/e917cdec9e574505be1ae86ec3a4f00c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60efd75c377b4d7ba483614a13050ea6] 2024-11-17T22:41:24,536 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d7781f43beb24bbe8510ad840efe3fc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731883279024 2024-11-17T22:41:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:24,536 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee5353d54fe240be978b9df926e20813, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731883279024 2024-11-17T22:41:24,536 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting dcafcb6095024c438ead2f8ea9e98de5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731883279062 2024-11-17T22:41:24,536 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting e917cdec9e574505be1ae86ec3a4f00c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731883279062 2024-11-17T22:41:24,537 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60efd75c377b4d7ba483614a13050ea6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731883281229 2024-11-17T22:41:24,537 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:24,537 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ee062b1ac94be0b046a008f8f2d2f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731883281229 2024-11-17T22:41:24,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:41:24,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,550 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#B#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:24,550 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:24,551 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/4081ddc9ce3b47e5bf9bbae864bcfaa4 is 50, key is test_row_0/B:col10/1731883281235/Put/seqid=0 2024-11-17T22:41:24,557 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117b1dd06247e084e65a2a5679eb53a47aa_dd1e0e7a755612e5d2f97d7c99dd3f57 store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:24,559 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117b1dd06247e084e65a2a5679eb53a47aa_dd1e0e7a755612e5d2f97d7c99dd3f57, store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:24,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117512d2b6e4a1c4d7787d3360050227ff6_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883283353/Put/seqid=0 2024-11-17T22:41:24,559 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b1dd06247e084e65a2a5679eb53a47aa_dd1e0e7a755612e5d2f97d7c99dd3f57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:24,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742178_1354 (size=12104) 2024-11-17T22:41:24,577 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/4081ddc9ce3b47e5bf9bbae864bcfaa4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/4081ddc9ce3b47e5bf9bbae864bcfaa4 2024-11-17T22:41:24,582 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/B of dd1e0e7a755612e5d2f97d7c99dd3f57 into 4081ddc9ce3b47e5bf9bbae864bcfaa4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:24,582 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:24,582 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/B, priority=13, startTime=1731883284533; duration=0sec 2024-11-17T22:41:24,582 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:24,582 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:B 2024-11-17T22:41:24,582 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:24,583 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:24,583 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/C is initiating minor compaction (all files) 2024-11-17T22:41:24,583 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/C in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,583 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/a28314c892664526ba982cd6703dde3f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/c97e86ba38454c0b8725d2578eca0de9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/4e20bd70c8134eb4b4417cfd2b319fbf] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=35.2 K 2024-11-17T22:41:24,583 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting a28314c892664526ba982cd6703dde3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731883279024 2024-11-17T22:41:24,584 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting c97e86ba38454c0b8725d2578eca0de9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731883279062 2024-11-17T22:41:24,584 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e20bd70c8134eb4b4417cfd2b319fbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731883281229 2024-11-17T22:41:24,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742179_1355 (size=4469) 2024-11-17T22:41:24,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742180_1356 (size=14594) 2024-11-17T22:41:24,594 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#C#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:24,594 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/9f9e9a7513ca4a48aff0509e7ab6422e is 50, key is test_row_0/C:col10/1731883281235/Put/seqid=0 2024-11-17T22:41:24,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742181_1357 (size=12104) 2024-11-17T22:41:24,602 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/9f9e9a7513ca4a48aff0509e7ab6422e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/9f9e9a7513ca4a48aff0509e7ab6422e 2024-11-17T22:41:24,608 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/C of dd1e0e7a755612e5d2f97d7c99dd3f57 into 9f9e9a7513ca4a48aff0509e7ab6422e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:24,608 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:24,608 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/C, priority=13, startTime=1731883284534; duration=0sec 2024-11-17T22:41:24,608 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:24,608 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:C 2024-11-17T22:41:24,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883344632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:24,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:24,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:24,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883344747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:24,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:24,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:24,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:24,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:24,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883344956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:24,991 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#A#compaction#306 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:24,991 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:24,991 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/028bc1a0e79244919b5c699d76060587 is 175, key is test_row_0/A:col10/1731883281235/Put/seqid=0 2024-11-17T22:41:24,995 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117512d2b6e4a1c4d7787d3360050227ff6_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117512d2b6e4a1c4d7787d3360050227ff6_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:24,997 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/60c16ece905347e1b9d5e989fe5bc8e4, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:24,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/60c16ece905347e1b9d5e989fe5bc8e4 is 175, key is test_row_0/A:col10/1731883283353/Put/seqid=0 2024-11-17T22:41:24,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742182_1358 (size=31058) 2024-11-17T22:41:25,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742183_1359 (size=39549) 2024-11-17T22:41:25,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,009 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/028bc1a0e79244919b5c699d76060587 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/028bc1a0e79244919b5c699d76060587 2024-11-17T22:41:25,015 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/A of dd1e0e7a755612e5d2f97d7c99dd3f57 into 028bc1a0e79244919b5c699d76060587(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:25,015 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:25,015 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/A, priority=13, startTime=1731883284533; duration=0sec 2024-11-17T22:41:25,015 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:25,015 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:A 2024-11-17T22:41:25,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:25,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:25,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883345267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:25,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:25,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:25,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T22:41:25,403 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/60c16ece905347e1b9d5e989fe5bc8e4 2024-11-17T22:41:25,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/65e39b05fab5473188b28723a3226867 is 50, key is test_row_0/B:col10/1731883283353/Put/seqid=0 2024-11-17T22:41:25,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742184_1360 (size=12001) 2024-11-17T22:41:25,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883345417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:25,421 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:25,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883345426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:25,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883345426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:25,434 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:25,434 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:25,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883345447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:25,454 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:25,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:25,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:25,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:25,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883345776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:25,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/65e39b05fab5473188b28723a3226867 2024-11-17T22:41:25,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/7e0111989fed4d7080a7d8c9548978c8 is 50, key is test_row_0/C:col10/1731883283353/Put/seqid=0 2024-11-17T22:41:25,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742185_1361 (size=12001) 2024-11-17T22:41:25,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/7e0111989fed4d7080a7d8c9548978c8 2024-11-17T22:41:25,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/60c16ece905347e1b9d5e989fe5bc8e4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60c16ece905347e1b9d5e989fe5bc8e4 2024-11-17T22:41:25,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60c16ece905347e1b9d5e989fe5bc8e4, entries=200, sequenceid=77, filesize=38.6 K 2024-11-17T22:41:25,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/65e39b05fab5473188b28723a3226867 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/65e39b05fab5473188b28723a3226867 2024-11-17T22:41:25,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/65e39b05fab5473188b28723a3226867, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T22:41:25,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/7e0111989fed4d7080a7d8c9548978c8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/7e0111989fed4d7080a7d8c9548978c8 2024-11-17T22:41:25,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/7e0111989fed4d7080a7d8c9548978c8, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T22:41:25,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1303ms, sequenceid=77, compaction requested=false 2024-11-17T22:41:25,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:25,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T22:41:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:25,923 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:41:25,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e51fa71a3a2a4319a71a9340da8a334e_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883284621/Put/seqid=0 2024-11-17T22:41:25,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742186_1362 (size=12154) 2024-11-17T22:41:26,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:26,347 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e51fa71a3a2a4319a71a9340da8a334e_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e51fa71a3a2a4319a71a9340da8a334e_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/c35ec401f72e466da44587b2b2df6477, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/c35ec401f72e466da44587b2b2df6477 is 175, key is test_row_0/A:col10/1731883284621/Put/seqid=0 2024-11-17T22:41:26,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742187_1363 (size=30955) 2024-11-17T22:41:26,767 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/c35ec401f72e466da44587b2b2df6477 2024-11-17T22:41:26,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/1fe1cda2eb1145d9a2d3afb6b2458fac is 50, key is test_row_0/B:col10/1731883284621/Put/seqid=0 2024-11-17T22:41:26,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742188_1364 (size=12001) 2024-11-17T22:41:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:26,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:26,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883346921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:27,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883347037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:27,176 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/1fe1cda2eb1145d9a2d3afb6b2458fac 2024-11-17T22:41:27,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/3ccd1cebbc97431a9cd98b60c7f21749 is 50, key is test_row_0/C:col10/1731883284621/Put/seqid=0 2024-11-17T22:41:27,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742189_1365 (size=12001) 2024-11-17T22:41:27,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883347247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T22:41:27,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883347566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:27,593 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/3ccd1cebbc97431a9cd98b60c7f21749 2024-11-17T22:41:27,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/c35ec401f72e466da44587b2b2df6477 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/c35ec401f72e466da44587b2b2df6477 2024-11-17T22:41:27,600 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/c35ec401f72e466da44587b2b2df6477, entries=150, sequenceid=92, filesize=30.2 K 2024-11-17T22:41:27,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/1fe1cda2eb1145d9a2d3afb6b2458fac as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/1fe1cda2eb1145d9a2d3afb6b2458fac 2024-11-17T22:41:27,604 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/1fe1cda2eb1145d9a2d3afb6b2458fac, entries=150, sequenceid=92, filesize=11.7 K 2024-11-17T22:41:27,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/3ccd1cebbc97431a9cd98b60c7f21749 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/3ccd1cebbc97431a9cd98b60c7f21749 2024-11-17T22:41:27,608 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/3ccd1cebbc97431a9cd98b60c7f21749, entries=150, sequenceid=92, filesize=11.7 K 2024-11-17T22:41:27,609 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1686ms, sequenceid=92, compaction requested=true 2024-11-17T22:41:27,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:27,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:27,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-17T22:41:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-17T22:41:27,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-17T22:41:27,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4400 sec 2024-11-17T22:41:27,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 4.4440 sec 2024-11-17T22:41:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:28,091 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:28,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:41:28,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173bbf51f8752a419f87c196843395d997_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883286909/Put/seqid=0 2024-11-17T22:41:28,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742190_1366 (size=14594) 2024-11-17T22:41:28,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883348149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:28,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883348256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:28,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883348467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:28,502 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:28,505 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173bbf51f8752a419f87c196843395d997_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173bbf51f8752a419f87c196843395d997_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:28,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/563bbdeac6a14020994f437c20809b0a, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:28,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/563bbdeac6a14020994f437c20809b0a is 175, key is test_row_0/A:col10/1731883286909/Put/seqid=0 2024-11-17T22:41:28,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742191_1367 (size=39549) 2024-11-17T22:41:28,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T22:41:28,618 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T22:41:28,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883348776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:28,919 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/563bbdeac6a14020994f437c20809b0a 2024-11-17T22:41:28,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/bbc9d7aee1b34309a66bbc525fef9490 is 50, key is test_row_0/B:col10/1731883286909/Put/seqid=0 2024-11-17T22:41:28,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742192_1368 (size=12001) 2024-11-17T22:41:29,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883349287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:29,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/bbc9d7aee1b34309a66bbc525fef9490 2024-11-17T22:41:29,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/dcb1f155829242f086f3fae2342b09ad is 50, key is test_row_0/C:col10/1731883286909/Put/seqid=0 2024-11-17T22:41:29,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742193_1369 (size=12001) 2024-11-17T22:41:29,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49802 deadline: 1731883349436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:29,443 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:29,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49774 deadline: 1731883349477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:29,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49820 deadline: 1731883349477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:29,481 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:29,482 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:29,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1731883349497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:29,503 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in dd1e0e7a755612e5d2f97d7c99dd3f57 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:29,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/dcb1f155829242f086f3fae2342b09ad 2024-11-17T22:41:29,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/563bbdeac6a14020994f437c20809b0a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/563bbdeac6a14020994f437c20809b0a 2024-11-17T22:41:29,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/563bbdeac6a14020994f437c20809b0a, entries=200, sequenceid=117, filesize=38.6 K 2024-11-17T22:41:29,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/bbc9d7aee1b34309a66bbc525fef9490 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/bbc9d7aee1b34309a66bbc525fef9490 2024-11-17T22:41:29,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/bbc9d7aee1b34309a66bbc525fef9490, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T22:41:29,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/dcb1f155829242f086f3fae2342b09ad as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/dcb1f155829242f086f3fae2342b09ad 2024-11-17T22:41:29,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/dcb1f155829242f086f3fae2342b09ad, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T22:41:29,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1666ms, sequenceid=117, compaction requested=true 2024-11-17T22:41:29,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:29,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:29,758 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:41:29,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:29,758 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:41:29,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:29,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:29,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:29,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:29,760 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:41:29,760 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:41:29,760 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/A is initiating minor compaction (all files) 2024-11-17T22:41:29,760 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/B is initiating minor compaction (all files) 2024-11-17T22:41:29,760 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/B in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:29,760 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/A in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:29,760 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/4081ddc9ce3b47e5bf9bbae864bcfaa4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/65e39b05fab5473188b28723a3226867, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/1fe1cda2eb1145d9a2d3afb6b2458fac, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/bbc9d7aee1b34309a66bbc525fef9490] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=47.0 K 2024-11-17T22:41:29,760 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/028bc1a0e79244919b5c699d76060587, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60c16ece905347e1b9d5e989fe5bc8e4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/c35ec401f72e466da44587b2b2df6477, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/563bbdeac6a14020994f437c20809b0a] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=137.8 K 2024-11-17T22:41:29,760 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:29,760 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/028bc1a0e79244919b5c699d76060587, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60c16ece905347e1b9d5e989fe5bc8e4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/c35ec401f72e466da44587b2b2df6477, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/563bbdeac6a14020994f437c20809b0a] 2024-11-17T22:41:29,760 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4081ddc9ce3b47e5bf9bbae864bcfaa4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731883281229 2024-11-17T22:41:29,761 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 028bc1a0e79244919b5c699d76060587, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731883281229 2024-11-17T22:41:29,761 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 65e39b05fab5473188b28723a3226867, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731883283353 2024-11-17T22:41:29,761 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60c16ece905347e1b9d5e989fe5bc8e4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731883283353 2024-11-17T22:41:29,761 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fe1cda2eb1145d9a2d3afb6b2458fac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731883284601 2024-11-17T22:41:29,761 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting c35ec401f72e466da44587b2b2df6477, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731883284601 2024-11-17T22:41:29,761 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc9d7aee1b34309a66bbc525fef9490, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731883286909 2024-11-17T22:41:29,762 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 563bbdeac6a14020994f437c20809b0a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731883286880 2024-11-17T22:41:29,769 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#B#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:29,770 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/699a88e32c8b4cf88b60ffacd97d6063 is 50, key is test_row_0/B:col10/1731883286909/Put/seqid=0 2024-11-17T22:41:29,776 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:29,778 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411179e0bf92740dd4d57b8ea56012ccca647_dd1e0e7a755612e5d2f97d7c99dd3f57 store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:29,779 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411179e0bf92740dd4d57b8ea56012ccca647_dd1e0e7a755612e5d2f97d7c99dd3f57, store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:29,779 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179e0bf92740dd4d57b8ea56012ccca647_dd1e0e7a755612e5d2f97d7c99dd3f57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:29,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742194_1370 (size=12241) 2024-11-17T22:41:29,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742195_1371 (size=4469) 2024-11-17T22:41:29,792 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#A#compaction#318 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:29,792 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/6be497456121497f9fca48446a64e485 is 175, key is test_row_0/A:col10/1731883286909/Put/seqid=0 2024-11-17T22:41:29,794 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/699a88e32c8b4cf88b60ffacd97d6063 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/699a88e32c8b4cf88b60ffacd97d6063 2024-11-17T22:41:29,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742196_1372 (size=31195) 2024-11-17T22:41:29,799 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/B of dd1e0e7a755612e5d2f97d7c99dd3f57 into 699a88e32c8b4cf88b60ffacd97d6063(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:29,799 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:29,799 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/B, priority=12, startTime=1731883289758; duration=0sec 2024-11-17T22:41:29,799 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:29,799 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:B 2024-11-17T22:41:29,799 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:41:29,800 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:41:29,800 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/C is initiating minor compaction (all files) 2024-11-17T22:41:29,800 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/C in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:29,800 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/9f9e9a7513ca4a48aff0509e7ab6422e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/7e0111989fed4d7080a7d8c9548978c8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/3ccd1cebbc97431a9cd98b60c7f21749, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/dcb1f155829242f086f3fae2342b09ad] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=47.0 K 2024-11-17T22:41:29,800 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f9e9a7513ca4a48aff0509e7ab6422e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731883281229 2024-11-17T22:41:29,800 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e0111989fed4d7080a7d8c9548978c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731883283353 2024-11-17T22:41:29,801 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ccd1cebbc97431a9cd98b60c7f21749, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731883284601 2024-11-17T22:41:29,801 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb1f155829242f086f3fae2342b09ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731883286909 2024-11-17T22:41:29,803 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/6be497456121497f9fca48446a64e485 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/6be497456121497f9fca48446a64e485 2024-11-17T22:41:29,806 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/A of dd1e0e7a755612e5d2f97d7c99dd3f57 into 6be497456121497f9fca48446a64e485(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:29,806 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:29,807 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/A, priority=12, startTime=1731883289757; duration=0sec 2024-11-17T22:41:29,807 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:29,807 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:A 2024-11-17T22:41:29,810 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#C#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:29,811 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/109aa2bec4ef41858fd780819ab0fbac is 50, key is test_row_0/C:col10/1731883286909/Put/seqid=0 2024-11-17T22:41:29,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742197_1373 (size=12241) 2024-11-17T22:41:29,822 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/109aa2bec4ef41858fd780819ab0fbac as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/109aa2bec4ef41858fd780819ab0fbac 2024-11-17T22:41:29,826 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/C of dd1e0e7a755612e5d2f97d7c99dd3f57 into 109aa2bec4ef41858fd780819ab0fbac(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:29,826 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:29,826 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/C, priority=12, startTime=1731883289758; duration=0sec 2024-11-17T22:41:29,826 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:29,826 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:C 2024-11-17T22:41:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:30,311 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:30,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:41:30,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b6353bc89e1642f39ddfeb7eb64db641_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883290309/Put/seqid=0 2024-11-17T22:41:30,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742198_1374 (size=14644) 2024-11-17T22:41:30,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:30,325 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b6353bc89e1642f39ddfeb7eb64db641_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b6353bc89e1642f39ddfeb7eb64db641_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:30,326 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/5fa7abc34b4046a5af7c19845eb98ad1, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:30,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/5fa7abc34b4046a5af7c19845eb98ad1 is 175, key is test_row_0/A:col10/1731883290309/Put/seqid=0 2024-11-17T22:41:30,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742199_1375 (size=39599) 2024-11-17T22:41:30,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883350442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:30,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883350556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:30,732 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/5fa7abc34b4046a5af7c19845eb98ad1 2024-11-17T22:41:30,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/a66efdb69d8e4fe9a77933836da71b5c is 50, key is test_row_0/B:col10/1731883290309/Put/seqid=0 2024-11-17T22:41:30,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742200_1376 (size=12051) 2024-11-17T22:41:30,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883350767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:31,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883351076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:31,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/a66efdb69d8e4fe9a77933836da71b5c 2024-11-17T22:41:31,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/b0c3ac3fbce841e6b0c9e1a0f9628c6a is 50, key is test_row_0/C:col10/1731883290309/Put/seqid=0 2024-11-17T22:41:31,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742201_1377 (size=12051) 2024-11-17T22:41:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T22:41:31,367 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:31,367 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-17T22:41:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T22:41:31,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:31,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:31,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T22:41:31,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:31,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/b0c3ac3fbce841e6b0c9e1a0f9628c6a 2024-11-17T22:41:31,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/5fa7abc34b4046a5af7c19845eb98ad1 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/5fa7abc34b4046a5af7c19845eb98ad1 2024-11-17T22:41:31,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/5fa7abc34b4046a5af7c19845eb98ad1, entries=200, sequenceid=132, filesize=38.7 K 2024-11-17T22:41:31,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/a66efdb69d8e4fe9a77933836da71b5c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a66efdb69d8e4fe9a77933836da71b5c 2024-11-17T22:41:31,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a66efdb69d8e4fe9a77933836da71b5c, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T22:41:31,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/b0c3ac3fbce841e6b0c9e1a0f9628c6a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/b0c3ac3fbce841e6b0c9e1a0f9628c6a 2024-11-17T22:41:31,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/b0c3ac3fbce841e6b0c9e1a0f9628c6a, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T22:41:31,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1265ms, sequenceid=132, compaction requested=false 2024-11-17T22:41:31,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:31,600 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:31,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:41:31,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117db0aca969fbd480e9f095f736af63c81_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883290430/Put/seqid=0 2024-11-17T22:41:31,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742202_1378 (size=14794) 2024-11-17T22:41:31,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883351644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:31,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:31,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:31,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:31,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:31,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T22:41:31,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883351757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:31,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:31,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883351966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:31,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:31,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T22:41:32,011 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:32,015 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117db0aca969fbd480e9f095f736af63c81_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117db0aca969fbd480e9f095f736af63c81_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:32,016 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/8dfb1cf3ed384628ae4eb78e8f1cc9c2, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:32,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/8dfb1cf3ed384628ae4eb78e8f1cc9c2 is 175, key is test_row_0/A:col10/1731883290430/Put/seqid=0 2024-11-17T22:41:32,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742203_1379 (size=39749) 2024-11-17T22:41:32,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:32,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:32,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883352277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:32,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,426 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/8dfb1cf3ed384628ae4eb78e8f1cc9c2 2024-11-17T22:41:32,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/a0cfccd9aee44a279a0ed8f297c21b72 is 50, key is test_row_0/B:col10/1731883290430/Put/seqid=0 2024-11-17T22:41:32,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742204_1380 (size=12151) 2024-11-17T22:41:32,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:32,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:32,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T22:41:32,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:32,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:32,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883352797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:32,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/a0cfccd9aee44a279a0ed8f297c21b72 2024-11-17T22:41:32,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/377ba581999748baaf8477c458e3610c is 50, key is test_row_0/C:col10/1731883290430/Put/seqid=0 2024-11-17T22:41:32,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742205_1381 (size=12151) 2024-11-17T22:41:32,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:32,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:33,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:33,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:33,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:33,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:33,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:33,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:33,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:33,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:33,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:33,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/377ba581999748baaf8477c458e3610c 2024-11-17T22:41:33,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/8dfb1cf3ed384628ae4eb78e8f1cc9c2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/8dfb1cf3ed384628ae4eb78e8f1cc9c2 2024-11-17T22:41:33,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/8dfb1cf3ed384628ae4eb78e8f1cc9c2, entries=200, sequenceid=157, filesize=38.8 K 2024-11-17T22:41:33,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/a0cfccd9aee44a279a0ed8f297c21b72 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a0cfccd9aee44a279a0ed8f297c21b72 2024-11-17T22:41:33,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a0cfccd9aee44a279a0ed8f297c21b72, entries=150, sequenceid=157, filesize=11.9 K 2024-11-17T22:41:33,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/377ba581999748baaf8477c458e3610c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/377ba581999748baaf8477c458e3610c 2024-11-17T22:41:33,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/377ba581999748baaf8477c458e3610c, entries=150, sequenceid=157, filesize=11.9 K 2024-11-17T22:41:33,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1662ms, sequenceid=157, compaction requested=true 2024-11-17T22:41:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:33,263 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:33,263 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:33,264 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:33,264 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:33,264 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/A is initiating minor compaction (all files) 2024-11-17T22:41:33,264 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/B is initiating minor compaction (all files) 2024-11-17T22:41:33,264 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/B in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:33,264 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/A in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:33,264 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/6be497456121497f9fca48446a64e485, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/5fa7abc34b4046a5af7c19845eb98ad1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/8dfb1cf3ed384628ae4eb78e8f1cc9c2] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=108.0 K 2024-11-17T22:41:33,264 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/699a88e32c8b4cf88b60ffacd97d6063, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a66efdb69d8e4fe9a77933836da71b5c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a0cfccd9aee44a279a0ed8f297c21b72] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=35.6 K 2024-11-17T22:41:33,264 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:33,264 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/6be497456121497f9fca48446a64e485, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/5fa7abc34b4046a5af7c19845eb98ad1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/8dfb1cf3ed384628ae4eb78e8f1cc9c2] 2024-11-17T22:41:33,264 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 699a88e32c8b4cf88b60ffacd97d6063, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731883286909 2024-11-17T22:41:33,264 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6be497456121497f9fca48446a64e485, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731883286909 2024-11-17T22:41:33,265 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting a66efdb69d8e4fe9a77933836da71b5c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731883288130 2024-11-17T22:41:33,265 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fa7abc34b4046a5af7c19845eb98ad1, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731883288130 2024-11-17T22:41:33,265 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting a0cfccd9aee44a279a0ed8f297c21b72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883290428 2024-11-17T22:41:33,265 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dfb1cf3ed384628ae4eb78e8f1cc9c2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883290428 2024-11-17T22:41:33,271 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:33,272 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#B#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:33,273 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/80e6dafcdf5247389ae60ace6c9342c4 is 50, key is test_row_0/B:col10/1731883290430/Put/seqid=0 2024-11-17T22:41:33,275 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411174b4f483cf571427c8a57ca7a7b4b569d_dd1e0e7a755612e5d2f97d7c99dd3f57 store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:33,277 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411174b4f483cf571427c8a57ca7a7b4b569d_dd1e0e7a755612e5d2f97d7c99dd3f57, store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:33,277 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174b4f483cf571427c8a57ca7a7b4b569d_dd1e0e7a755612e5d2f97d7c99dd3f57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:33,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742206_1382 (size=12493) 2024-11-17T22:41:33,293 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/80e6dafcdf5247389ae60ace6c9342c4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/80e6dafcdf5247389ae60ace6c9342c4 2024-11-17T22:41:33,297 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/B of dd1e0e7a755612e5d2f97d7c99dd3f57 into 80e6dafcdf5247389ae60ace6c9342c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:33,297 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:33,297 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/B, priority=13, startTime=1731883293263; duration=0sec 2024-11-17T22:41:33,298 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:33,298 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:B 2024-11-17T22:41:33,298 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:33,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742207_1383 (size=4469) 2024-11-17T22:41:33,299 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:33,299 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#A#compaction#327 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:33,299 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/C is initiating minor compaction (all files) 2024-11-17T22:41:33,299 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/C in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:33,299 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/109aa2bec4ef41858fd780819ab0fbac, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/b0c3ac3fbce841e6b0c9e1a0f9628c6a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/377ba581999748baaf8477c458e3610c] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=35.6 K 2024-11-17T22:41:33,300 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/018c615de4654bfb86cce298859cddef is 175, key is test_row_0/A:col10/1731883290430/Put/seqid=0 2024-11-17T22:41:33,300 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 109aa2bec4ef41858fd780819ab0fbac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731883286909 2024-11-17T22:41:33,300 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b0c3ac3fbce841e6b0c9e1a0f9628c6a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731883288130 2024-11-17T22:41:33,301 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 377ba581999748baaf8477c458e3610c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883290428 2024-11-17T22:41:33,308 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#C#compaction#328 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:33,308 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/a9a8843ad0574bd3aa02b93c3da7ce53 is 50, key is test_row_0/C:col10/1731883290430/Put/seqid=0 2024-11-17T22:41:33,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742209_1385 (size=12493) 2024-11-17T22:41:33,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742208_1384 (size=31447) 2024-11-17T22:41:33,320 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/018c615de4654bfb86cce298859cddef as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/018c615de4654bfb86cce298859cddef 2024-11-17T22:41:33,325 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/A of dd1e0e7a755612e5d2f97d7c99dd3f57 into 018c615de4654bfb86cce298859cddef(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:33,325 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:33,325 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/A, priority=13, startTime=1731883293263; duration=0sec 2024-11-17T22:41:33,325 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:33,325 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:A 2024-11-17T22:41:33,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T22:41:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:33,353 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:41:33,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170a99913c569a47559386f145d788221e_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883291643/Put/seqid=0 2024-11-17T22:41:33,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742210_1386 (size=12304) 2024-11-17T22:41:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T22:41:33,718 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/a9a8843ad0574bd3aa02b93c3da7ce53 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/a9a8843ad0574bd3aa02b93c3da7ce53 2024-11-17T22:41:33,722 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/C of dd1e0e7a755612e5d2f97d7c99dd3f57 into a9a8843ad0574bd3aa02b93c3da7ce53(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:33,722 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:33,722 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/C, priority=13, startTime=1731883293263; duration=0sec 2024-11-17T22:41:33,722 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:33,722 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:C 2024-11-17T22:41:33,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:33,776 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170a99913c569a47559386f145d788221e_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170a99913c569a47559386f145d788221e_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/530e1ebb81ef491aa95893f64279b72f, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:33,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/530e1ebb81ef491aa95893f64279b72f is 175, key is test_row_0/A:col10/1731883291643/Put/seqid=0 2024-11-17T22:41:33,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742211_1387 (size=31105) 2024-11-17T22:41:33,790 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/530e1ebb81ef491aa95893f64279b72f 2024-11-17T22:41:33,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/13eb3c38b80446b18adac53d8189bc3d is 50, key is test_row_0/B:col10/1731883291643/Put/seqid=0 2024-11-17T22:41:33,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742212_1388 (size=12151) 2024-11-17T22:41:33,817 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/13eb3c38b80446b18adac53d8189bc3d 2024-11-17T22:41:33,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:33,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/8b4aaf68627147918ebe40b9eecef2bc is 50, key is test_row_0/C:col10/1731883291643/Put/seqid=0 2024-11-17T22:41:33,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742213_1389 (size=12151) 2024-11-17T22:41:33,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883353964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:34,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883354077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:34,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:34,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:34,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:34,231 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/8b4aaf68627147918ebe40b9eecef2bc 2024-11-17T22:41:34,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/530e1ebb81ef491aa95893f64279b72f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/530e1ebb81ef491aa95893f64279b72f 2024-11-17T22:41:34,237 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/530e1ebb81ef491aa95893f64279b72f, entries=150, sequenceid=170, filesize=30.4 K 2024-11-17T22:41:34,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/13eb3c38b80446b18adac53d8189bc3d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/13eb3c38b80446b18adac53d8189bc3d 2024-11-17T22:41:34,241 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/13eb3c38b80446b18adac53d8189bc3d, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T22:41:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/8b4aaf68627147918ebe40b9eecef2bc as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/8b4aaf68627147918ebe40b9eecef2bc 2024-11-17T22:41:34,244 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/8b4aaf68627147918ebe40b9eecef2bc, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T22:41:34,245 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 892ms, sequenceid=170, compaction requested=false 2024-11-17T22:41:34,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:34,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:34,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-17T22:41:34,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-17T22:41:34,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-17T22:41:34,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8770 sec 2024-11-17T22:41:34,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.8810 sec 2024-11-17T22:41:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:34,292 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:34,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T22:41:34,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111779c1be1157ab4721a77625627c49a121_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883293963/Put/seqid=0 2024-11-17T22:41:34,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742214_1390 (size=14794) 2024-11-17T22:41:34,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883354333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:34,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883354447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:34,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883354656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:34,713 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:34,716 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111779c1be1157ab4721a77625627c49a121_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779c1be1157ab4721a77625627c49a121_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:34,717 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/3d766f3131bd4d4aba6947263b864d0d, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:34,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/3d766f3131bd4d4aba6947263b864d0d is 175, key is test_row_0/A:col10/1731883293963/Put/seqid=0 2024-11-17T22:41:34,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742215_1391 (size=39749) 2024-11-17T22:41:34,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883354967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:35,128 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/3d766f3131bd4d4aba6947263b864d0d 2024-11-17T22:41:35,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/10d3704c94564211b9c68af31f02f52a is 50, key is test_row_0/B:col10/1731883293963/Put/seqid=0 2024-11-17T22:41:35,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742216_1392 (size=12151) 2024-11-17T22:41:35,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/10d3704c94564211b9c68af31f02f52a 2024-11-17T22:41:35,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/561f22c29616458da1668c6205fe877b is 50, key is test_row_0/C:col10/1731883293963/Put/seqid=0 2024-11-17T22:41:35,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742217_1393 (size=12151) 2024-11-17T22:41:35,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:35,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883355476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:35,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T22:41:35,537 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:35,538 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-17T22:41:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T22:41:35,539 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:35,540 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:35,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:35,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/561f22c29616458da1668c6205fe877b 2024-11-17T22:41:35,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/3d766f3131bd4d4aba6947263b864d0d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/3d766f3131bd4d4aba6947263b864d0d 2024-11-17T22:41:35,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/3d766f3131bd4d4aba6947263b864d0d, entries=200, sequenceid=197, filesize=38.8 K 2024-11-17T22:41:35,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/10d3704c94564211b9c68af31f02f52a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/10d3704c94564211b9c68af31f02f52a 2024-11-17T22:41:35,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/10d3704c94564211b9c68af31f02f52a, entries=150, sequenceid=197, filesize=11.9 K 2024-11-17T22:41:35,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/561f22c29616458da1668c6205fe877b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/561f22c29616458da1668c6205fe877b 2024-11-17T22:41:35,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/561f22c29616458da1668c6205fe877b, entries=150, sequenceid=197, filesize=11.9 K 2024-11-17T22:41:35,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1271ms, sequenceid=197, compaction requested=true 2024-11-17T22:41:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:35,564 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:35,564 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:35,564 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:35,564 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:35,565 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/B is initiating minor compaction (all files) 2024-11-17T22:41:35,565 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/A is initiating minor compaction (all files) 2024-11-17T22:41:35,565 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/A in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:35,565 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/B in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:35,565 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/80e6dafcdf5247389ae60ace6c9342c4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/13eb3c38b80446b18adac53d8189bc3d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/10d3704c94564211b9c68af31f02f52a] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=35.9 K 2024-11-17T22:41:35,565 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/018c615de4654bfb86cce298859cddef, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/530e1ebb81ef491aa95893f64279b72f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/3d766f3131bd4d4aba6947263b864d0d] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=99.9 K 2024-11-17T22:41:35,565 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:35,565 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/018c615de4654bfb86cce298859cddef, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/530e1ebb81ef491aa95893f64279b72f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/3d766f3131bd4d4aba6947263b864d0d] 2024-11-17T22:41:35,565 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 80e6dafcdf5247389ae60ace6c9342c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883290428 2024-11-17T22:41:35,565 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 018c615de4654bfb86cce298859cddef, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883290428 2024-11-17T22:41:35,565 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 13eb3c38b80446b18adac53d8189bc3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731883291618 2024-11-17T22:41:35,565 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 530e1ebb81ef491aa95893f64279b72f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731883291618 2024-11-17T22:41:35,565 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 10d3704c94564211b9c68af31f02f52a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731883293957 2024-11-17T22:41:35,566 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d766f3131bd4d4aba6947263b864d0d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731883293944 2024-11-17T22:41:35,571 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:35,572 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#B#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:35,572 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/d597c40fb39c4f92be584d27084dd71e is 50, key is test_row_0/B:col10/1731883293963/Put/seqid=0 2024-11-17T22:41:35,572 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117f725b6e5b0344c5ba35e65801385c045_dd1e0e7a755612e5d2f97d7c99dd3f57 store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:35,574 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117f725b6e5b0344c5ba35e65801385c045_dd1e0e7a755612e5d2f97d7c99dd3f57, store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:35,574 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f725b6e5b0344c5ba35e65801385c045_dd1e0e7a755612e5d2f97d7c99dd3f57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:35,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742218_1394 (size=12595) 2024-11-17T22:41:35,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742219_1395 (size=4469) 2024-11-17T22:41:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T22:41:35,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-17T22:41:35,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:35,692 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T22:41:35,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111779dcd2aad3ea477b86b41c0d4db266e9_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883294331/Put/seqid=0 2024-11-17T22:41:35,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742220_1396 (size=12304) 2024-11-17T22:41:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T22:41:35,978 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#A#compaction#336 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:35,978 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/985167d2b44a4e79a3134aadbd41902d is 175, key is test_row_0/A:col10/1731883293963/Put/seqid=0 2024-11-17T22:41:35,979 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/d597c40fb39c4f92be584d27084dd71e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/d597c40fb39c4f92be584d27084dd71e 2024-11-17T22:41:35,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742221_1397 (size=31549) 2024-11-17T22:41:35,983 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/B of dd1e0e7a755612e5d2f97d7c99dd3f57 into d597c40fb39c4f92be584d27084dd71e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:35,984 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:35,984 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/B, priority=13, startTime=1731883295564; duration=0sec 2024-11-17T22:41:35,984 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:35,984 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:B 2024-11-17T22:41:35,984 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:35,984 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:35,985 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/C is initiating minor compaction (all files) 2024-11-17T22:41:35,985 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/C in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:35,985 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/a9a8843ad0574bd3aa02b93c3da7ce53, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/8b4aaf68627147918ebe40b9eecef2bc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/561f22c29616458da1668c6205fe877b] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=35.9 K 2024-11-17T22:41:35,985 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a8843ad0574bd3aa02b93c3da7ce53, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883290428 2024-11-17T22:41:35,985 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b4aaf68627147918ebe40b9eecef2bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731883291618 2024-11-17T22:41:35,986 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 561f22c29616458da1668c6205fe877b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731883293957 2024-11-17T22:41:35,994 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#C#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:35,995 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/2c6ceadfc745488f8eae65c86d295c59 is 50, key is test_row_0/C:col10/1731883293963/Put/seqid=0 2024-11-17T22:41:35,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742222_1398 (size=12595) 2024-11-17T22:41:36,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:36,106 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111779dcd2aad3ea477b86b41c0d4db266e9_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779dcd2aad3ea477b86b41c0d4db266e9_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:36,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/b7080581c38743d9b889c86a51d46e48, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:36,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/b7080581c38743d9b889c86a51d46e48 is 175, key is test_row_0/A:col10/1731883294331/Put/seqid=0 2024-11-17T22:41:36,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742223_1399 (size=31105) 2024-11-17T22:41:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T22:41:36,385 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/985167d2b44a4e79a3134aadbd41902d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/985167d2b44a4e79a3134aadbd41902d 2024-11-17T22:41:36,389 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/A of dd1e0e7a755612e5d2f97d7c99dd3f57 into 985167d2b44a4e79a3134aadbd41902d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:36,389 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:36,389 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/A, priority=13, startTime=1731883295564; duration=0sec 2024-11-17T22:41:36,389 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:36,389 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:A 2024-11-17T22:41:36,402 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/2c6ceadfc745488f8eae65c86d295c59 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/2c6ceadfc745488f8eae65c86d295c59 2024-11-17T22:41:36,405 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/C of dd1e0e7a755612e5d2f97d7c99dd3f57 into 2c6ceadfc745488f8eae65c86d295c59(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:36,405 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:36,405 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/C, priority=13, startTime=1731883295564; duration=0sec 2024-11-17T22:41:36,405 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:36,405 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:C 2024-11-17T22:41:36,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:36,511 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/b7080581c38743d9b889c86a51d46e48 2024-11-17T22:41:36,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/4c122b09d3064b3b8f56fea405e45046 is 50, key is test_row_0/B:col10/1731883294331/Put/seqid=0 2024-11-17T22:41:36,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742224_1400 (size=12151) 2024-11-17T22:41:36,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883356622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T22:41:36,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883356737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:36,926 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/4c122b09d3064b3b8f56fea405e45046 2024-11-17T22:41:36,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/1fba3e5d708b499d8de1ec3a26b58fc3 is 50, key is test_row_0/C:col10/1731883294331/Put/seqid=0 2024-11-17T22:41:36,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742225_1401 (size=12151) 2024-11-17T22:41:36,936 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/1fba3e5d708b499d8de1ec3a26b58fc3 2024-11-17T22:41:36,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/b7080581c38743d9b889c86a51d46e48 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/b7080581c38743d9b889c86a51d46e48 2024-11-17T22:41:36,942 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/b7080581c38743d9b889c86a51d46e48, entries=150, sequenceid=207, filesize=30.4 K 2024-11-17T22:41:36,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/4c122b09d3064b3b8f56fea405e45046 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/4c122b09d3064b3b8f56fea405e45046 2024-11-17T22:41:36,946 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/4c122b09d3064b3b8f56fea405e45046, entries=150, sequenceid=207, filesize=11.9 K 2024-11-17T22:41:36,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/1fba3e5d708b499d8de1ec3a26b58fc3 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/1fba3e5d708b499d8de1ec3a26b58fc3 2024-11-17T22:41:36,949 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/1fba3e5d708b499d8de1ec3a26b58fc3, entries=150, sequenceid=207, filesize=11.9 K 2024-11-17T22:41:36,950 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1258ms, sequenceid=207, compaction requested=false 2024-11-17T22:41:36,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:36,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:36,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-17T22:41:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-17T22:41:36,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-17T22:41:36,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4110 sec 2024-11-17T22:41:36,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4150 sec 2024-11-17T22:41:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:36,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:36,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-17T22:41:36,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177cbd342d21ca4f03978423c182dffc72_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883296955/Put/seqid=0 2024-11-17T22:41:36,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742226_1402 (size=14794) 2024-11-17T22:41:37,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883356996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:37,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883357107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:37,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:37,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:37,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:37,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883357327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:37,367 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:37,370 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177cbd342d21ca4f03978423c182dffc72_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177cbd342d21ca4f03978423c182dffc72_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:37,371 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/4123ef1ee1b54527aea218c31e917622, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:37,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/4123ef1ee1b54527aea218c31e917622 is 175, key is test_row_0/A:col10/1731883296955/Put/seqid=0 2024-11-17T22:41:37,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742227_1403 (size=39749) 2024-11-17T22:41:37,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883357637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T22:41:37,697 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:37,697 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-17T22:41:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T22:41:37,699 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:37,699 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:37,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:37,775 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/4123ef1ee1b54527aea218c31e917622 2024-11-17T22:41:37,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/5f5ad9683bd44b7ba969007ed31958d0 is 50, key is test_row_0/B:col10/1731883296955/Put/seqid=0 2024-11-17T22:41:37,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742228_1404 (size=12151) 2024-11-17T22:41:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T22:41:37,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T22:41:37,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:37,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:37,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:37,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:37,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T22:41:38,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:38,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T22:41:38,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T22:41:38,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:38,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1731883358157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:38,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:38,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:38,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:41:38,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/5f5ad9683bd44b7ba969007ed31958d0 2024-11-17T22:41:38,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/48179ecace6243cf9ce980966380d7a8 is 50, key is test_row_0/C:col10/1731883296955/Put/seqid=0 2024-11-17T22:41:38,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742229_1405 (size=12151) 2024-11-17T22:41:38,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T22:41:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:38,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T22:41:38,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T22:41:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/48179ecace6243cf9ce980966380d7a8 2024-11-17T22:41:38,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/4123ef1ee1b54527aea218c31e917622 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/4123ef1ee1b54527aea218c31e917622 2024-11-17T22:41:38,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/4123ef1ee1b54527aea218c31e917622, entries=200, sequenceid=237, filesize=38.8 K 2024-11-17T22:41:38,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/5f5ad9683bd44b7ba969007ed31958d0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/5f5ad9683bd44b7ba969007ed31958d0 2024-11-17T22:41:38,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/5f5ad9683bd44b7ba969007ed31958d0, entries=150, sequenceid=237, filesize=11.9 K 2024-11-17T22:41:38,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/48179ecace6243cf9ce980966380d7a8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/48179ecace6243cf9ce980966380d7a8 2024-11-17T22:41:38,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T22:41:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/48179ecace6243cf9ce980966380d7a8, entries=150, sequenceid=237, filesize=11.9 K 2024-11-17T22:41:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:38,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1659ms, sequenceid=237, compaction requested=true 2024-11-17T22:41:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:38,616 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:38,616 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd1e0e7a755612e5d2f97d7c99dd3f57:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:38,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:38,617 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:38,617 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/A is initiating minor compaction (all files) 2024-11-17T22:41:38,617 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/A in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,617 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/985167d2b44a4e79a3134aadbd41902d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/b7080581c38743d9b889c86a51d46e48, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/4123ef1ee1b54527aea218c31e917622] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=100.0 K 2024-11-17T22:41:38,617 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,617 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/985167d2b44a4e79a3134aadbd41902d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/b7080581c38743d9b889c86a51d46e48, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/4123ef1ee1b54527aea218c31e917622] 2024-11-17T22:41:38,617 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:38,617 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/B is initiating minor compaction (all files) 2024-11-17T22:41:38,617 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/B in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,618 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/d597c40fb39c4f92be584d27084dd71e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/4c122b09d3064b3b8f56fea405e45046, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/5f5ad9683bd44b7ba969007ed31958d0] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=36.0 K 2024-11-17T22:41:38,618 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 985167d2b44a4e79a3134aadbd41902d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731883293957 2024-11-17T22:41:38,618 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d597c40fb39c4f92be584d27084dd71e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731883293957 2024-11-17T22:41:38,618 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7080581c38743d9b889c86a51d46e48, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731883294308 2024-11-17T22:41:38,618 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c122b09d3064b3b8f56fea405e45046, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731883294308 2024-11-17T22:41:38,618 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4123ef1ee1b54527aea218c31e917622, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731883296596 2024-11-17T22:41:38,618 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5ad9683bd44b7ba969007ed31958d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731883296610 2024-11-17T22:41:38,623 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:38,625 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:38,625 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/00bef9775e4d4905ad89e87a5887c881 is 50, key is test_row_0/B:col10/1731883296955/Put/seqid=0 2024-11-17T22:41:38,629 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411178bc71c9a85a4447aa43f8327083291e0_dd1e0e7a755612e5d2f97d7c99dd3f57 store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:38,631 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411178bc71c9a85a4447aa43f8327083291e0_dd1e0e7a755612e5d2f97d7c99dd3f57, store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:38,631 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178bc71c9a85a4447aa43f8327083291e0_dd1e0e7a755612e5d2f97d7c99dd3f57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:38,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742230_1406 (size=12697) 2024-11-17T22:41:38,637 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/00bef9775e4d4905ad89e87a5887c881 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/00bef9775e4d4905ad89e87a5887c881 2024-11-17T22:41:38,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742231_1407 (size=4469) 2024-11-17T22:41:38,642 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/B of dd1e0e7a755612e5d2f97d7c99dd3f57 into 00bef9775e4d4905ad89e87a5887c881(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:38,642 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:38,642 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/B, priority=13, startTime=1731883298616; duration=0sec 2024-11-17T22:41:38,642 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:38,642 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:B 2024-11-17T22:41:38,642 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:38,644 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:38,644 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): dd1e0e7a755612e5d2f97d7c99dd3f57/C is initiating minor compaction (all files) 2024-11-17T22:41:38,644 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd1e0e7a755612e5d2f97d7c99dd3f57/C in TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,644 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/2c6ceadfc745488f8eae65c86d295c59, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/1fba3e5d708b499d8de1ec3a26b58fc3, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/48179ecace6243cf9ce980966380d7a8] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp, totalSize=36.0 K 2024-11-17T22:41:38,644 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c6ceadfc745488f8eae65c86d295c59, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731883293957 2024-11-17T22:41:38,645 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fba3e5d708b499d8de1ec3a26b58fc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731883294308 2024-11-17T22:41:38,645 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 48179ecace6243cf9ce980966380d7a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731883296610 2024-11-17T22:41:38,652 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#C#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:38,653 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/508eddf5a0a94e3a8b2559f0438c35ba is 50, key is test_row_0/C:col10/1731883296955/Put/seqid=0 2024-11-17T22:41:38,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742232_1408 (size=12697) 2024-11-17T22:41:38,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T22:41:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:38,767 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-17T22:41:38,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ced3a89128eb48a8abf19b5e7a40284a_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883296961/Put/seqid=0 2024-11-17T22:41:38,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742233_1409 (size=12304) 2024-11-17T22:41:38,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:38,787 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ced3a89128eb48a8abf19b5e7a40284a_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ced3a89128eb48a8abf19b5e7a40284a_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:38,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/8b2207a87f664da8867ff3177f79a18b, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:38,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/8b2207a87f664da8867ff3177f79a18b is 175, key is test_row_0/A:col10/1731883296961/Put/seqid=0 2024-11-17T22:41:38,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742234_1410 (size=31105) 2024-11-17T22:41:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T22:41:39,013 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-11-17T22:41:39,013 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:39,013 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,014 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:39,015 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-17T22:41:39,015 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:39,015 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,015 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:39,015 INFO [Thread-1564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1564. 2024-11-17T22:41:39,015 DEBUG [Thread-1564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:39,015 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,015 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:39,018 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-11-17T22:41:39,018 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:39,018 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,018 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:39,019 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-11-17T22:41:39,019 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:39,019 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,019 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:39,042 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd1e0e7a755612e5d2f97d7c99dd3f57#A#compaction#344 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:39,042 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/a0e557c92edb4580b824c4d024a6a51f is 175, key is test_row_0/A:col10/1731883296955/Put/seqid=0 2024-11-17T22:41:39,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742235_1411 (size=31651) 2024-11-17T22:41:39,064 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/508eddf5a0a94e3a8b2559f0438c35ba as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/508eddf5a0a94e3a8b2559f0438c35ba 2024-11-17T22:41:39,067 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/C of dd1e0e7a755612e5d2f97d7c99dd3f57 into 508eddf5a0a94e3a8b2559f0438c35ba(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:39,067 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:39,067 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/C, priority=13, startTime=1731883298616; duration=0sec 2024-11-17T22:41:39,067 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:39,067 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:C 2024-11-17T22:41:39,181 INFO [Thread-1545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1545. 2024-11-17T22:41:39,181 DEBUG [Thread-1545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:39,181 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,182 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:39,193 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/8b2207a87f664da8867ff3177f79a18b 2024-11-17T22:41:39,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/e4192d73a0a246d7a07c9719a56699a4 is 50, key is test_row_0/B:col10/1731883296961/Put/seqid=0 2024-11-17T22:41:39,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742236_1412 (size=12151) 2024-11-17T22:41:39,456 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/a0e557c92edb4580b824c4d024a6a51f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/a0e557c92edb4580b824c4d024a6a51f 2024-11-17T22:41:39,462 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd1e0e7a755612e5d2f97d7c99dd3f57/A of dd1e0e7a755612e5d2f97d7c99dd3f57 into a0e557c92edb4580b824c4d024a6a51f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:39,462 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:39,463 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., storeName=dd1e0e7a755612e5d2f97d7c99dd3f57/A, priority=13, startTime=1731883298616; duration=0sec 2024-11-17T22:41:39,463 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:39,463 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd1e0e7a755612e5d2f97d7c99dd3f57:A 2024-11-17T22:41:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. as already flushing 2024-11-17T22:41:39,509 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-17T22:41:39,509 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:39,509 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,510 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:39,519 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-17T22:41:39,519 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:39,519 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,520 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:39,540 INFO [Thread-1547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1547. 2024-11-17T22:41:39,540 DEBUG [Thread-1547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:39,540 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,540 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:39,581 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-17T22:41:39,582 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:41:39,582 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,582 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:39,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:39,610 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/e4192d73a0a246d7a07c9719a56699a4 2024-11-17T22:41:39,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/ddc5f6d70b6d4cd5aab1f559ebb4ebd4 is 50, key is test_row_0/C:col10/1731883296961/Put/seqid=0 2024-11-17T22:41:39,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742237_1413 (size=12151) 2024-11-17T22:41:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T22:41:40,027 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/ddc5f6d70b6d4cd5aab1f559ebb4ebd4 2024-11-17T22:41:40,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/8b2207a87f664da8867ff3177f79a18b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/8b2207a87f664da8867ff3177f79a18b 2024-11-17T22:41:40,036 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/8b2207a87f664da8867ff3177f79a18b, entries=150, sequenceid=247, filesize=30.4 K 2024-11-17T22:41:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/e4192d73a0a246d7a07c9719a56699a4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/e4192d73a0a246d7a07c9719a56699a4 2024-11-17T22:41:40,040 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/e4192d73a0a246d7a07c9719a56699a4, entries=150, sequenceid=247, filesize=11.9 K 2024-11-17T22:41:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/ddc5f6d70b6d4cd5aab1f559ebb4ebd4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/ddc5f6d70b6d4cd5aab1f559ebb4ebd4 2024-11-17T22:41:40,045 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/ddc5f6d70b6d4cd5aab1f559ebb4ebd4, entries=150, sequenceid=247, filesize=11.9 K 2024-11-17T22:41:40,047 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=33.54 KB/34350 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1279ms, sequenceid=247, compaction requested=false 2024-11-17T22:41:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for dd1e0e7a755612e5d2f97d7c99dd3f57: 2024-11-17T22:41:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-17T22:41:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-17T22:41:40,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-17T22:41:40,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3490 sec 2024-11-17T22:41:40,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.3530 sec 2024-11-17T22:41:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T22:41:41,858 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:41,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T22:41:41,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 159 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1490 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4463 rows 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1491 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4472 rows 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1500 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4499 rows 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1499 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4497 rows 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1502 2024-11-17T22:41:41,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4503 rows 2024-11-17T22:41:41,859 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T22:41:41,860 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T22:41:41,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:41,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:41,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:41:41,862 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T22:41:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T22:41:41,869 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883301868"}]},"ts":"1731883301868"} 2024-11-17T22:41:41,870 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T22:41:41,870 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T22:41:41,871 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T22:41:41,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, UNASSIGN}] 2024-11-17T22:41:41,874 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, UNASSIGN 2024-11-17T22:41:41,875 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=dd1e0e7a755612e5d2f97d7c99dd3f57, regionState=CLOSING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:41,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, UNASSIGN because future has completed 2024-11-17T22:41:41,876 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T22:41:41,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:41:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T22:41:42,029 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:42,029 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T22:41:42,029 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing dd1e0e7a755612e5d2f97d7c99dd3f57, disabling compactions & flushes 2024-11-17T22:41:42,029 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:42,029 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:42,029 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. after waiting 0 ms 2024-11-17T22:41:42,029 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:42,029 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing dd1e0e7a755612e5d2f97d7c99dd3f57 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T22:41:42,039 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176c7daeb43f8a4a1293e86d44dfc5e10d_dd1e0e7a755612e5d2f97d7c99dd3f57 is 50, key is test_row_0/A:col10/1731883299517/Put/seqid=0 2024-11-17T22:41:42,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742238_1414 (size=12304) 2024-11-17T22:41:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T22:41:42,444 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:42,450 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176c7daeb43f8a4a1293e86d44dfc5e10d_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176c7daeb43f8a4a1293e86d44dfc5e10d_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:42,452 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/14b14dd80ea9420d86646918152e8669, store: [table=TestAcidGuarantees family=A region=dd1e0e7a755612e5d2f97d7c99dd3f57] 2024-11-17T22:41:42,452 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/14b14dd80ea9420d86646918152e8669 is 175, key is test_row_0/A:col10/1731883299517/Put/seqid=0 2024-11-17T22:41:42,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742239_1415 (size=31105) 2024-11-17T22:41:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T22:41:42,858 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/14b14dd80ea9420d86646918152e8669 2024-11-17T22:41:42,870 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/be8bb054a858484eb910dbe45a03769b is 50, key is test_row_0/B:col10/1731883299517/Put/seqid=0 2024-11-17T22:41:42,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742240_1416 (size=12151) 2024-11-17T22:41:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T22:41:43,275 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/be8bb054a858484eb910dbe45a03769b 2024-11-17T22:41:43,288 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/d3b984448dee4e139199347309af62b0 is 50, key is test_row_0/C:col10/1731883299517/Put/seqid=0 2024-11-17T22:41:43,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742241_1417 (size=12151) 2024-11-17T22:41:43,693 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/d3b984448dee4e139199347309af62b0 2024-11-17T22:41:43,704 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/A/14b14dd80ea9420d86646918152e8669 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/14b14dd80ea9420d86646918152e8669 2024-11-17T22:41:43,710 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/14b14dd80ea9420d86646918152e8669, entries=150, sequenceid=257, filesize=30.4 K 2024-11-17T22:41:43,711 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/B/be8bb054a858484eb910dbe45a03769b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/be8bb054a858484eb910dbe45a03769b 2024-11-17T22:41:43,715 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/be8bb054a858484eb910dbe45a03769b, entries=150, sequenceid=257, filesize=11.9 K 2024-11-17T22:41:43,716 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/.tmp/C/d3b984448dee4e139199347309af62b0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/d3b984448dee4e139199347309af62b0 2024-11-17T22:41:43,720 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/d3b984448dee4e139199347309af62b0, entries=150, sequenceid=257, filesize=11.9 K 2024-11-17T22:41:43,721 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1692ms, sequenceid=257, compaction requested=true 2024-11-17T22:41:43,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/ee5353d54fe240be978b9df926e20813, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/e917cdec9e574505be1ae86ec3a4f00c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60efd75c377b4d7ba483614a13050ea6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/028bc1a0e79244919b5c699d76060587, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60c16ece905347e1b9d5e989fe5bc8e4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/c35ec401f72e466da44587b2b2df6477, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/563bbdeac6a14020994f437c20809b0a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/6be497456121497f9fca48446a64e485, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/5fa7abc34b4046a5af7c19845eb98ad1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/8dfb1cf3ed384628ae4eb78e8f1cc9c2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/018c615de4654bfb86cce298859cddef, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/530e1ebb81ef491aa95893f64279b72f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/3d766f3131bd4d4aba6947263b864d0d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/985167d2b44a4e79a3134aadbd41902d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/b7080581c38743d9b889c86a51d46e48, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/4123ef1ee1b54527aea218c31e917622] to archive 2024-11-17T22:41:43,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:41:43,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/ee5353d54fe240be978b9df926e20813 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/ee5353d54fe240be978b9df926e20813 2024-11-17T22:41:43,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/e917cdec9e574505be1ae86ec3a4f00c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/e917cdec9e574505be1ae86ec3a4f00c 2024-11-17T22:41:43,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60efd75c377b4d7ba483614a13050ea6 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60efd75c377b4d7ba483614a13050ea6 2024-11-17T22:41:43,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/028bc1a0e79244919b5c699d76060587 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/028bc1a0e79244919b5c699d76060587 2024-11-17T22:41:43,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60c16ece905347e1b9d5e989fe5bc8e4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/60c16ece905347e1b9d5e989fe5bc8e4 2024-11-17T22:41:43,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/c35ec401f72e466da44587b2b2df6477 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/c35ec401f72e466da44587b2b2df6477 2024-11-17T22:41:43,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/563bbdeac6a14020994f437c20809b0a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/563bbdeac6a14020994f437c20809b0a 2024-11-17T22:41:43,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/6be497456121497f9fca48446a64e485 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/6be497456121497f9fca48446a64e485 2024-11-17T22:41:43,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/5fa7abc34b4046a5af7c19845eb98ad1 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/5fa7abc34b4046a5af7c19845eb98ad1 2024-11-17T22:41:43,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/8dfb1cf3ed384628ae4eb78e8f1cc9c2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/8dfb1cf3ed384628ae4eb78e8f1cc9c2 2024-11-17T22:41:43,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/018c615de4654bfb86cce298859cddef to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/018c615de4654bfb86cce298859cddef 2024-11-17T22:41:43,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/530e1ebb81ef491aa95893f64279b72f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/530e1ebb81ef491aa95893f64279b72f 2024-11-17T22:41:43,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/3d766f3131bd4d4aba6947263b864d0d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/3d766f3131bd4d4aba6947263b864d0d 2024-11-17T22:41:43,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/985167d2b44a4e79a3134aadbd41902d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/985167d2b44a4e79a3134aadbd41902d 2024-11-17T22:41:43,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/b7080581c38743d9b889c86a51d46e48 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/b7080581c38743d9b889c86a51d46e48 2024-11-17T22:41:43,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/4123ef1ee1b54527aea218c31e917622 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/4123ef1ee1b54527aea218c31e917622 2024-11-17T22:41:43,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/d7781f43beb24bbe8510ad840efe3fc3, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/dcafcb6095024c438ead2f8ea9e98de5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/4081ddc9ce3b47e5bf9bbae864bcfaa4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/92ee062b1ac94be0b046a008f8f2d2f5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/65e39b05fab5473188b28723a3226867, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/1fe1cda2eb1145d9a2d3afb6b2458fac, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/699a88e32c8b4cf88b60ffacd97d6063, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/bbc9d7aee1b34309a66bbc525fef9490, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a66efdb69d8e4fe9a77933836da71b5c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/80e6dafcdf5247389ae60ace6c9342c4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a0cfccd9aee44a279a0ed8f297c21b72, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/13eb3c38b80446b18adac53d8189bc3d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/d597c40fb39c4f92be584d27084dd71e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/10d3704c94564211b9c68af31f02f52a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/4c122b09d3064b3b8f56fea405e45046, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/5f5ad9683bd44b7ba969007ed31958d0] to archive 2024-11-17T22:41:43,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:41:43,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/d7781f43beb24bbe8510ad840efe3fc3 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/d7781f43beb24bbe8510ad840efe3fc3 2024-11-17T22:41:43,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/dcafcb6095024c438ead2f8ea9e98de5 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/dcafcb6095024c438ead2f8ea9e98de5 2024-11-17T22:41:43,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/4081ddc9ce3b47e5bf9bbae864bcfaa4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/4081ddc9ce3b47e5bf9bbae864bcfaa4 2024-11-17T22:41:43,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/92ee062b1ac94be0b046a008f8f2d2f5 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/92ee062b1ac94be0b046a008f8f2d2f5 2024-11-17T22:41:43,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/65e39b05fab5473188b28723a3226867 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/65e39b05fab5473188b28723a3226867 2024-11-17T22:41:43,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/1fe1cda2eb1145d9a2d3afb6b2458fac to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/1fe1cda2eb1145d9a2d3afb6b2458fac 2024-11-17T22:41:43,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/699a88e32c8b4cf88b60ffacd97d6063 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/699a88e32c8b4cf88b60ffacd97d6063 2024-11-17T22:41:43,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/bbc9d7aee1b34309a66bbc525fef9490 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/bbc9d7aee1b34309a66bbc525fef9490 2024-11-17T22:41:43,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a66efdb69d8e4fe9a77933836da71b5c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a66efdb69d8e4fe9a77933836da71b5c 2024-11-17T22:41:43,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/80e6dafcdf5247389ae60ace6c9342c4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/80e6dafcdf5247389ae60ace6c9342c4 2024-11-17T22:41:43,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a0cfccd9aee44a279a0ed8f297c21b72 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/a0cfccd9aee44a279a0ed8f297c21b72 2024-11-17T22:41:43,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/13eb3c38b80446b18adac53d8189bc3d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/13eb3c38b80446b18adac53d8189bc3d 2024-11-17T22:41:43,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/d597c40fb39c4f92be584d27084dd71e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/d597c40fb39c4f92be584d27084dd71e 2024-11-17T22:41:43,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/10d3704c94564211b9c68af31f02f52a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/10d3704c94564211b9c68af31f02f52a 2024-11-17T22:41:43,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/4c122b09d3064b3b8f56fea405e45046 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/4c122b09d3064b3b8f56fea405e45046 2024-11-17T22:41:43,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/5f5ad9683bd44b7ba969007ed31958d0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/5f5ad9683bd44b7ba969007ed31958d0 2024-11-17T22:41:43,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/a28314c892664526ba982cd6703dde3f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/c97e86ba38454c0b8725d2578eca0de9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/9f9e9a7513ca4a48aff0509e7ab6422e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/4e20bd70c8134eb4b4417cfd2b319fbf, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/7e0111989fed4d7080a7d8c9548978c8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/3ccd1cebbc97431a9cd98b60c7f21749, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/109aa2bec4ef41858fd780819ab0fbac, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/dcb1f155829242f086f3fae2342b09ad, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/b0c3ac3fbce841e6b0c9e1a0f9628c6a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/a9a8843ad0574bd3aa02b93c3da7ce53, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/377ba581999748baaf8477c458e3610c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/8b4aaf68627147918ebe40b9eecef2bc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/2c6ceadfc745488f8eae65c86d295c59, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/561f22c29616458da1668c6205fe877b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/1fba3e5d708b499d8de1ec3a26b58fc3, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/48179ecace6243cf9ce980966380d7a8] to archive 2024-11-17T22:41:43,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:41:43,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/a28314c892664526ba982cd6703dde3f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/a28314c892664526ba982cd6703dde3f 2024-11-17T22:41:43,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/c97e86ba38454c0b8725d2578eca0de9 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/c97e86ba38454c0b8725d2578eca0de9 2024-11-17T22:41:43,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/9f9e9a7513ca4a48aff0509e7ab6422e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/9f9e9a7513ca4a48aff0509e7ab6422e 2024-11-17T22:41:43,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/4e20bd70c8134eb4b4417cfd2b319fbf to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/4e20bd70c8134eb4b4417cfd2b319fbf 2024-11-17T22:41:43,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/7e0111989fed4d7080a7d8c9548978c8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/7e0111989fed4d7080a7d8c9548978c8 2024-11-17T22:41:43,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/3ccd1cebbc97431a9cd98b60c7f21749 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/3ccd1cebbc97431a9cd98b60c7f21749 2024-11-17T22:41:43,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/109aa2bec4ef41858fd780819ab0fbac to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/109aa2bec4ef41858fd780819ab0fbac 2024-11-17T22:41:43,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/dcb1f155829242f086f3fae2342b09ad to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/dcb1f155829242f086f3fae2342b09ad 2024-11-17T22:41:43,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/b0c3ac3fbce841e6b0c9e1a0f9628c6a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/b0c3ac3fbce841e6b0c9e1a0f9628c6a 2024-11-17T22:41:43,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/a9a8843ad0574bd3aa02b93c3da7ce53 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/a9a8843ad0574bd3aa02b93c3da7ce53 2024-11-17T22:41:43,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/377ba581999748baaf8477c458e3610c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/377ba581999748baaf8477c458e3610c 2024-11-17T22:41:43,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/8b4aaf68627147918ebe40b9eecef2bc to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/8b4aaf68627147918ebe40b9eecef2bc 2024-11-17T22:41:43,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/2c6ceadfc745488f8eae65c86d295c59 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/2c6ceadfc745488f8eae65c86d295c59 2024-11-17T22:41:43,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/561f22c29616458da1668c6205fe877b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/561f22c29616458da1668c6205fe877b 2024-11-17T22:41:43,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/1fba3e5d708b499d8de1ec3a26b58fc3 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/1fba3e5d708b499d8de1ec3a26b58fc3 2024-11-17T22:41:43,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/48179ecace6243cf9ce980966380d7a8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/48179ecace6243cf9ce980966380d7a8 2024-11-17T22:41:43,774 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/recovered.edits/260.seqid, newMaxSeqId=260, maxSeqId=4 2024-11-17T22:41:43,775 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. 2024-11-17T22:41:43,775 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for dd1e0e7a755612e5d2f97d7c99dd3f57: Waiting for close lock at 1731883302029Running coprocessor pre-close hooks at 1731883302029Disabling compacts and flushes for region at 1731883302029Disabling writes for close at 1731883302029Obtaining lock to block concurrent updates at 1731883302029Preparing flush snapshotting stores in dd1e0e7a755612e5d2f97d7c99dd3f57 at 1731883302029Finished memstore snapshotting TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731883302030 (+1 ms)Flushing stores of TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57. at 1731883302031 (+1 ms)Flushing dd1e0e7a755612e5d2f97d7c99dd3f57/A: creating writer at 1731883302031Flushing mob file dd1e0e7a755612e5d2f97d7c99dd3f57/A: appending metadata at 1731883302039 (+8 ms)Flushing mob file dd1e0e7a755612e5d2f97d7c99dd3f57/A: closing flushed file at 1731883302039Flushing dd1e0e7a755612e5d2f97d7c99dd3f57/A: appending metadata at 1731883302452 (+413 ms)Flushing dd1e0e7a755612e5d2f97d7c99dd3f57/A: closing flushed file at 1731883302452Flushing dd1e0e7a755612e5d2f97d7c99dd3f57/B: creating writer at 1731883302865 (+413 ms)Flushing dd1e0e7a755612e5d2f97d7c99dd3f57/B: appending metadata at 1731883302869 (+4 ms)Flushing dd1e0e7a755612e5d2f97d7c99dd3f57/B: closing flushed file at 1731883302869Flushing dd1e0e7a755612e5d2f97d7c99dd3f57/C: creating writer at 1731883303285 (+416 ms)Flushing dd1e0e7a755612e5d2f97d7c99dd3f57/C: appending metadata at 1731883303287 (+2 ms)Flushing dd1e0e7a755612e5d2f97d7c99dd3f57/C: closing flushed file at 1731883303288 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14321d44: reopening flushed file at 1731883303702 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15880546: reopening flushed file at 1731883303710 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@724b6042: reopening flushed file at 1731883303715 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for dd1e0e7a755612e5d2f97d7c99dd3f57 in 1692ms, sequenceid=257, compaction requested=true at 1731883303721 (+6 ms)Writing region close event to WAL at 1731883303772 (+51 ms)Running coprocessor post-close hooks at 1731883303775 (+3 ms)Closed at 1731883303775 2024-11-17T22:41:43,777 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:43,777 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=dd1e0e7a755612e5d2f97d7c99dd3f57, regionState=CLOSED 2024-11-17T22:41:43,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:41:43,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-17T22:41:43,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure dd1e0e7a755612e5d2f97d7c99dd3f57, server=1a6e40b21a48,43011,1731883188895 in 1.9030 sec 2024-11-17T22:41:43,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-17T22:41:43,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd1e0e7a755612e5d2f97d7c99dd3f57, UNASSIGN in 1.9070 sec 2024-11-17T22:41:43,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-17T22:41:43,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9110 sec 2024-11-17T22:41:43,785 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883303784"}]},"ts":"1731883303784"} 2024-11-17T22:41:43,786 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T22:41:43,786 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T22:41:43,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9240 sec 2024-11-17T22:41:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T22:41:44,018 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:44,019 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T22:41:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:44,022 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T22:41:44,023 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:44,027 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,031 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/recovered.edits] 2024-11-17T22:41:44,034 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/14b14dd80ea9420d86646918152e8669 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/14b14dd80ea9420d86646918152e8669 2024-11-17T22:41:44,035 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/8b2207a87f664da8867ff3177f79a18b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/8b2207a87f664da8867ff3177f79a18b 2024-11-17T22:41:44,036 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/a0e557c92edb4580b824c4d024a6a51f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/A/a0e557c92edb4580b824c4d024a6a51f 2024-11-17T22:41:44,038 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/00bef9775e4d4905ad89e87a5887c881 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/00bef9775e4d4905ad89e87a5887c881 2024-11-17T22:41:44,039 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/be8bb054a858484eb910dbe45a03769b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/be8bb054a858484eb910dbe45a03769b 2024-11-17T22:41:44,039 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/e4192d73a0a246d7a07c9719a56699a4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/B/e4192d73a0a246d7a07c9719a56699a4 2024-11-17T22:41:44,041 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/508eddf5a0a94e3a8b2559f0438c35ba to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/508eddf5a0a94e3a8b2559f0438c35ba 2024-11-17T22:41:44,042 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/d3b984448dee4e139199347309af62b0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/d3b984448dee4e139199347309af62b0 2024-11-17T22:41:44,042 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/ddc5f6d70b6d4cd5aab1f559ebb4ebd4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/C/ddc5f6d70b6d4cd5aab1f559ebb4ebd4 2024-11-17T22:41:44,044 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/recovered.edits/260.seqid to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57/recovered.edits/260.seqid 2024-11-17T22:41:44,044 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,044 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T22:41:44,045 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T22:41:44,045 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T22:41:44,047 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170a99913c569a47559386f145d788221e_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170a99913c569a47559386f145d788221e_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,048 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171fa5ece54d3f48a9a8bab560a257b616_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171fa5ece54d3f48a9a8bab560a257b616_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,049 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173bbf51f8752a419f87c196843395d997_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173bbf51f8752a419f87c196843395d997_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,050 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117512d2b6e4a1c4d7787d3360050227ff6_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117512d2b6e4a1c4d7787d3360050227ff6_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,050 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176c7daeb43f8a4a1293e86d44dfc5e10d_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176c7daeb43f8a4a1293e86d44dfc5e10d_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,051 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779c1be1157ab4721a77625627c49a121_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779c1be1157ab4721a77625627c49a121_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,052 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779dcd2aad3ea477b86b41c0d4db266e9_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779dcd2aad3ea477b86b41c0d4db266e9_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,053 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177cbd342d21ca4f03978423c182dffc72_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177cbd342d21ca4f03978423c182dffc72_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,053 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b6353bc89e1642f39ddfeb7eb64db641_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b6353bc89e1642f39ddfeb7eb64db641_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,054 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c9fc37f645ca4e4c881ee02b63cd68f5_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c9fc37f645ca4e4c881ee02b63cd68f5_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,055 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ce6488eb7c32463f8ef9bbb375825341_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ce6488eb7c32463f8ef9bbb375825341_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,055 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ced3a89128eb48a8abf19b5e7a40284a_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ced3a89128eb48a8abf19b5e7a40284a_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,056 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117db0aca969fbd480e9f095f736af63c81_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117db0aca969fbd480e9f095f736af63c81_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,057 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e51fa71a3a2a4319a71a9340da8a334e_dd1e0e7a755612e5d2f97d7c99dd3f57 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e51fa71a3a2a4319a71a9340da8a334e_dd1e0e7a755612e5d2f97d7c99dd3f57 2024-11-17T22:41:44,057 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T22:41:44,059 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:44,061 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T22:41:44,063 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T22:41:44,064 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:44,064 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T22:41:44,064 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731883304064"}]},"ts":"9223372036854775807"} 2024-11-17T22:41:44,066 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T22:41:44,066 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => dd1e0e7a755612e5d2f97d7c99dd3f57, NAME => 'TestAcidGuarantees,,1731883275853.dd1e0e7a755612e5d2f97d7c99dd3f57.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T22:41:44,066 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T22:41:44,066 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731883304066"}]},"ts":"9223372036854775807"} 2024-11-17T22:41:44,068 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T22:41:44,068 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:44,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-17T22:41:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T22:41:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T22:41:44,130 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:44,135 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 240), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 438), ProcessCount=11 (was 11), AvailableMemoryMB=4445 (was 4619) 2024-11-17T22:41:44,143 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=4445 2024-11-17T22:41:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T22:41:44,145 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T22:41:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T22:41:44,147 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T22:41:44,147 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:44,147 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-17T22:41:44,148 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T22:41:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-17T22:41:44,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742242_1418 (size=959) 2024-11-17T22:41:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-17T22:41:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-17T22:41:44,557 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71 2024-11-17T22:41:44,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742243_1419 (size=53) 2024-11-17T22:41:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-17T22:41:44,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:41:44,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6c99de6ca45c57ddad88306e57b16183, disabling compactions & flushes 2024-11-17T22:41:44,968 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:44,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:44,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. after waiting 0 ms 2024-11-17T22:41:44,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:44,969 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:44,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6c99de6ca45c57ddad88306e57b16183: Waiting for close lock at 1731883304968Disabling compacts and flushes for region at 1731883304968Disabling writes for close at 1731883304969 (+1 ms)Writing region close event to WAL at 1731883304969Closed at 1731883304969 2024-11-17T22:41:44,971 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T22:41:44,971 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731883304971"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731883304971"}]},"ts":"1731883304971"} 2024-11-17T22:41:44,974 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T22:41:44,976 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T22:41:44,976 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883304976"}]},"ts":"1731883304976"} 2024-11-17T22:41:44,978 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T22:41:44,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c99de6ca45c57ddad88306e57b16183, ASSIGN}] 2024-11-17T22:41:44,980 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c99de6ca45c57ddad88306e57b16183, ASSIGN 2024-11-17T22:41:44,981 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c99de6ca45c57ddad88306e57b16183, ASSIGN; state=OFFLINE, location=1a6e40b21a48,43011,1731883188895; forceNewPlan=false, retain=false 2024-11-17T22:41:45,132 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=6c99de6ca45c57ddad88306e57b16183, regionState=OPENING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:45,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c99de6ca45c57ddad88306e57b16183, ASSIGN because future has completed 2024-11-17T22:41:45,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:41:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-17T22:41:45,293 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:45,294 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} 2024-11-17T22:41:45,295 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,296 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:41:45,296 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,296 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,298 INFO [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,301 INFO [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c99de6ca45c57ddad88306e57b16183 columnFamilyName A 2024-11-17T22:41:45,301 DEBUG [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:45,302 INFO [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] regionserver.HStore(327): Store=6c99de6ca45c57ddad88306e57b16183/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:41:45,302 INFO [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,304 INFO [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c99de6ca45c57ddad88306e57b16183 columnFamilyName B 2024-11-17T22:41:45,304 DEBUG [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:45,304 INFO [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] regionserver.HStore(327): Store=6c99de6ca45c57ddad88306e57b16183/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:41:45,305 INFO [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,306 INFO [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c99de6ca45c57ddad88306e57b16183 columnFamilyName C 2024-11-17T22:41:45,306 DEBUG [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:41:45,306 INFO [StoreOpener-6c99de6ca45c57ddad88306e57b16183-1 {}] regionserver.HStore(327): Store=6c99de6ca45c57ddad88306e57b16183/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:41:45,306 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,307 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,307 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,309 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,309 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,309 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T22:41:45,310 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,312 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T22:41:45,313 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 6c99de6ca45c57ddad88306e57b16183; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74300536, jitterRate=0.10716426372528076}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T22:41:45,313 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:45,313 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 6c99de6ca45c57ddad88306e57b16183: Running coprocessor pre-open hook at 1731883305296Writing region info on filesystem at 1731883305296Initializing all the Stores at 1731883305297 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883305297Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883305298 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883305298Cleaning up temporary data from old regions at 1731883305309 (+11 ms)Running coprocessor post-open hooks at 1731883305313 (+4 ms)Region opened successfully at 1731883305313 2024-11-17T22:41:45,314 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., pid=126, masterSystemTime=1731883305289 2024-11-17T22:41:45,316 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:45,316 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:45,317 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=6c99de6ca45c57ddad88306e57b16183, regionState=OPEN, openSeqNum=2, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:45,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:41:45,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-17T22:41:45,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 in 182 msec 2024-11-17T22:41:45,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-17T22:41:45,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c99de6ca45c57ddad88306e57b16183, ASSIGN in 342 msec 2024-11-17T22:41:45,324 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T22:41:45,324 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883305324"}]},"ts":"1731883305324"} 2024-11-17T22:41:45,326 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T22:41:45,333 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T22:41:45,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-11-17T22:41:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-17T22:41:46,299 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:46,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dacd8f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:46,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:46,304 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:46,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:46,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:46,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301c5812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:46,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:46,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:46,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:46,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b0fde7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,307 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:46,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:46,311 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T22:41:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a6e40b21a48,45671,1731883188234 2024-11-17T22:41:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6b7371c2 2024-11-17T22:41:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T22:41:46,313 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T22:41:46,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72240ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:46,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:46,317 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:46,319 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad745e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:46,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:46,321 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f5f695, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:46,323 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21931399, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:46,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:46,325 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500cf906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:46,326 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4faf861f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:46,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:46,329 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e256845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:46,330 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30814cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:46,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:46,333 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18704cfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:46,336 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f58cb75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:46,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:46,338 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af107f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:46,340 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be2083d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:46,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:46,342 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c91addc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:46,343 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2bab3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:46,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:46,345 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd03700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:46,346 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58047960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:46,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:46,348 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab6c5d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:46,349 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7831a592, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:41:46,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:41:46,351 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:41:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:41:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:41:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c94886, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:41:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:41:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:41:46,352 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:41:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fb8cbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:41:46,353 DEBUG [Thread-1903 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:46,353 DEBUG [Thread-1899 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:46,353 DEBUG [Thread-1888 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:46,353 DEBUG [Thread-1890 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:46,353 DEBUG [Thread-1892 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:46,354 DEBUG [Thread-1884 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:46,354 DEBUG [Thread-1886 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:46,355 DEBUG [Thread-1897 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:46,355 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:46,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:46,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:46,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:46,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:46,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:46,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:46,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:46,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:46,356 DEBUG [Thread-1895 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-17T22:41:46,357 DEBUG [Thread-1901 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:41:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T22:41:46,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:41:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:41:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:41:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:41:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:41:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:41:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:41:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:41:46,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:41:46,361 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:46,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:41:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:41:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:41:46,362 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:46,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:46,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2] 2024-11-17T22:41:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:46,369 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:46,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:41:46,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/becf4610e8764765a8293f5a20013aca is 50, key is test_row_0/A:col10/1731883306363/Put/seqid=0 2024-11-17T22:41:46,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742244_1420 (size=14341) 2024-11-17T22:41:46,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883366400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883366400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883366403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883366402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883366404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T22:41:46,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T22:41:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:46,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883366517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883366517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883366517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883366517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883366526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T22:41:46,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:46,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:46,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:46,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T22:41:46,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883366727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883366727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883366727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883366737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883366737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:46,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/becf4610e8764765a8293f5a20013aca 2024-11-17T22:41:46,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/ac84bc361d814a7498af2938252a3731 is 50, key is test_row_0/B:col10/1731883306363/Put/seqid=0 2024-11-17T22:41:46,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742245_1421 (size=12001) 2024-11-17T22:41:46,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T22:41:46,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:46,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:46,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:46,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T22:41:46,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:46,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:46,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:46,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T22:41:47,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883367037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:47,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:47,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883367037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:47,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:47,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883367037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:47,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883367046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:47,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883367046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:47,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T22:41:47,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:47,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:47,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:47,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/ac84bc361d814a7498af2938252a3731 2024-11-17T22:41:47,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/00ceb06971164db3adb485bc33263836 is 50, key is test_row_0/C:col10/1731883306363/Put/seqid=0 2024-11-17T22:41:47,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742246_1422 (size=12001) 2024-11-17T22:41:47,246 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T22:41:47,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T22:41:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:47,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T22:41:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:47,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T22:41:47,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883367546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:47,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883367546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:47,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883367557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:47,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883367557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:47,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883367567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:47,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T22:41:47,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:47,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:47,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:47,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:47,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/00ceb06971164db3adb485bc33263836 2024-11-17T22:41:47,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/becf4610e8764765a8293f5a20013aca as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/becf4610e8764765a8293f5a20013aca 2024-11-17T22:41:47,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/becf4610e8764765a8293f5a20013aca, entries=200, sequenceid=13, filesize=14.0 K 2024-11-17T22:41:47,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/ac84bc361d814a7498af2938252a3731 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/ac84bc361d814a7498af2938252a3731 2024-11-17T22:41:47,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/ac84bc361d814a7498af2938252a3731, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T22:41:47,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/00ceb06971164db3adb485bc33263836 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/00ceb06971164db3adb485bc33263836 2024-11-17T22:41:47,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/00ceb06971164db3adb485bc33263836, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T22:41:47,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6c99de6ca45c57ddad88306e57b16183 in 1285ms, sequenceid=13, compaction requested=false 2024-11-17T22:41:47,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:47,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T22:41:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:47,734 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:41:47,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/f8a349f209c649d392f7c5f3f528639f is 50, key is test_row_0/A:col10/1731883306402/Put/seqid=0 2024-11-17T22:41:47,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742247_1423 (size=12001) 2024-11-17T22:41:48,142 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/f8a349f209c649d392f7c5f3f528639f 2024-11-17T22:41:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/a9c8ec5aaa524691b25d5feafc1e6a52 is 50, key is test_row_0/B:col10/1731883306402/Put/seqid=0 2024-11-17T22:41:48,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742248_1424 (size=12001) 2024-11-17T22:41:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T22:41:48,553 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/a9c8ec5aaa524691b25d5feafc1e6a52 2024-11-17T22:41:48,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/0b548e805c3f43e58d9cbff00f8fbd3e is 50, key is test_row_0/C:col10/1731883306402/Put/seqid=0 2024-11-17T22:41:48,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742249_1425 (size=12001) 2024-11-17T22:41:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:48,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883368583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883368586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883368587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883368587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,591 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883368588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883368696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883368697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883368697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883368697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883368907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883368907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883368907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883368908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:48,964 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/0b548e805c3f43e58d9cbff00f8fbd3e 2024-11-17T22:41:48,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/f8a349f209c649d392f7c5f3f528639f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f8a349f209c649d392f7c5f3f528639f 2024-11-17T22:41:48,971 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f8a349f209c649d392f7c5f3f528639f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T22:41:48,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/a9c8ec5aaa524691b25d5feafc1e6a52 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/a9c8ec5aaa524691b25d5feafc1e6a52 2024-11-17T22:41:48,975 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/a9c8ec5aaa524691b25d5feafc1e6a52, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T22:41:48,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/0b548e805c3f43e58d9cbff00f8fbd3e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/0b548e805c3f43e58d9cbff00f8fbd3e 2024-11-17T22:41:48,979 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/0b548e805c3f43e58d9cbff00f8fbd3e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T22:41:48,980 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6c99de6ca45c57ddad88306e57b16183 in 1246ms, sequenceid=37, compaction requested=false 2024-11-17T22:41:48,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:48,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:48,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-17T22:41:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-17T22:41:48,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-17T22:41:48,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6190 sec 2024-11-17T22:41:48,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.6280 sec 2024-11-17T22:41:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:49,224 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:49,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:41:49,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/1a4e21eaf9fc47088f862504aaaa655a is 50, key is test_row_0/A:col10/1731883308585/Put/seqid=0 2024-11-17T22:41:49,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742250_1426 (size=14341) 2024-11-17T22:41:49,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883369263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883369272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883369272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883369273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,296 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T22:41:49,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883369377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883369377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883369377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883369377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883369587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883369597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883369597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883369597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/1a4e21eaf9fc47088f862504aaaa655a 2024-11-17T22:41:49,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/164e337d229c4dee8842e6af5b2b1e5d is 50, key is test_row_0/B:col10/1731883308585/Put/seqid=0 2024-11-17T22:41:49,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742251_1427 (size=12001) 2024-11-17T22:41:49,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883369897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883369907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883369907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:49,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883369917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:50,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/164e337d229c4dee8842e6af5b2b1e5d 2024-11-17T22:41:50,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/c967ed3ab1f844eabacc8bc4c9c82a96 is 50, key is test_row_0/C:col10/1731883308585/Put/seqid=0 2024-11-17T22:41:50,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742252_1428 (size=12001) 2024-11-17T22:41:50,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:50,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883370407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:50,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:50,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883370417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:50,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883370417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:50,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883370427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:50,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:50,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/c967ed3ab1f844eabacc8bc4c9c82a96 2024-11-17T22:41:50,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/1a4e21eaf9fc47088f862504aaaa655a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/1a4e21eaf9fc47088f862504aaaa655a 2024-11-17T22:41:50,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/1a4e21eaf9fc47088f862504aaaa655a, entries=200, sequenceid=50, filesize=14.0 K 2024-11-17T22:41:50,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/164e337d229c4dee8842e6af5b2b1e5d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/164e337d229c4dee8842e6af5b2b1e5d 2024-11-17T22:41:50,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/164e337d229c4dee8842e6af5b2b1e5d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T22:41:50,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/c967ed3ab1f844eabacc8bc4c9c82a96 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c967ed3ab1f844eabacc8bc4c9c82a96 2024-11-17T22:41:50,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c967ed3ab1f844eabacc8bc4c9c82a96, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T22:41:50,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6c99de6ca45c57ddad88306e57b16183 in 1257ms, sequenceid=50, compaction requested=true 2024-11-17T22:41:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:50,481 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:50,481 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:50,482 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:50,482 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:50,482 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/B is initiating minor compaction (all files) 2024-11-17T22:41:50,482 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/A is initiating minor compaction (all files) 2024-11-17T22:41:50,482 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/B in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:50,482 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/A in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:50,482 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/ac84bc361d814a7498af2938252a3731, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/a9c8ec5aaa524691b25d5feafc1e6a52, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/164e337d229c4dee8842e6af5b2b1e5d] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=35.2 K 2024-11-17T22:41:50,482 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/becf4610e8764765a8293f5a20013aca, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f8a349f209c649d392f7c5f3f528639f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/1a4e21eaf9fc47088f862504aaaa655a] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=39.7 K 2024-11-17T22:41:50,482 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting ac84bc361d814a7498af2938252a3731, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731883306363 2024-11-17T22:41:50,482 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting becf4610e8764765a8293f5a20013aca, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731883306363 2024-11-17T22:41:50,482 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8a349f209c649d392f7c5f3f528639f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731883306400 2024-11-17T22:41:50,483 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c8ec5aaa524691b25d5feafc1e6a52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731883306400 2024-11-17T22:41:50,483 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a4e21eaf9fc47088f862504aaaa655a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883308584 2024-11-17T22:41:50,483 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 164e337d229c4dee8842e6af5b2b1e5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883308584 2024-11-17T22:41:50,489 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#A#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:50,489 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/614cdc28137a43c2add90f3cca6a94a9 is 50, key is test_row_0/A:col10/1731883308585/Put/seqid=0 2024-11-17T22:41:50,498 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:50,499 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/031fae5be3e649bf8bcea7d6c356c081 is 50, key is test_row_0/B:col10/1731883308585/Put/seqid=0 2024-11-17T22:41:50,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742253_1429 (size=12104) 2024-11-17T22:41:50,503 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/614cdc28137a43c2add90f3cca6a94a9 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/614cdc28137a43c2add90f3cca6a94a9 2024-11-17T22:41:50,507 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/A of 6c99de6ca45c57ddad88306e57b16183 into 614cdc28137a43c2add90f3cca6a94a9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:50,507 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:50,507 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/A, priority=13, startTime=1731883310481; duration=0sec 2024-11-17T22:41:50,507 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:50,507 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:A 2024-11-17T22:41:50,507 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:41:50,508 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:41:50,508 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/C is initiating minor compaction (all files) 2024-11-17T22:41:50,508 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/C in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:50,508 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/00ceb06971164db3adb485bc33263836, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/0b548e805c3f43e58d9cbff00f8fbd3e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c967ed3ab1f844eabacc8bc4c9c82a96] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=35.2 K 2024-11-17T22:41:50,509 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00ceb06971164db3adb485bc33263836, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731883306363 2024-11-17T22:41:50,509 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b548e805c3f43e58d9cbff00f8fbd3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731883306400 2024-11-17T22:41:50,509 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting c967ed3ab1f844eabacc8bc4c9c82a96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883308584 2024-11-17T22:41:50,515 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#C#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:50,515 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/04efb083e0d341d89001a87b747de180 is 50, key is test_row_0/C:col10/1731883308585/Put/seqid=0 2024-11-17T22:41:50,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742254_1430 (size=12104) 2024-11-17T22:41:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T22:41:50,517 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:50,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-17T22:41:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T22:41:50,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:50,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742255_1431 (size=12104) 2024-11-17T22:41:50,522 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:50,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:50,526 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/04efb083e0d341d89001a87b747de180 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/04efb083e0d341d89001a87b747de180 2024-11-17T22:41:50,530 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/C of 6c99de6ca45c57ddad88306e57b16183 into 04efb083e0d341d89001a87b747de180(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:50,530 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:50,530 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/C, priority=13, startTime=1731883310481; duration=0sec 2024-11-17T22:41:50,531 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:50,531 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:C 2024-11-17T22:41:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:50,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:41:50,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/83dbed8c6ecb4f72aafddb33c57df731 is 50, key is test_row_0/A:col10/1731883310612/Put/seqid=0 2024-11-17T22:41:50,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742256_1432 (size=14341) 2024-11-17T22:41:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T22:41:50,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883370666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:50,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T22:41:50,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:50,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:50,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:50,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:50,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:50,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883370776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:50,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T22:41:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:50,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T22:41:50,918 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/031fae5be3e649bf8bcea7d6c356c081 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/031fae5be3e649bf8bcea7d6c356c081 2024-11-17T22:41:50,921 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/B of 6c99de6ca45c57ddad88306e57b16183 into 031fae5be3e649bf8bcea7d6c356c081(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:50,921 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:50,921 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/B, priority=13, startTime=1731883310481; duration=0sec 2024-11-17T22:41:50,921 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:50,921 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:B 2024-11-17T22:41:50,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T22:41:50,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:50,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:50,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:50,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:50,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883370987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:51,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/83dbed8c6ecb4f72aafddb33c57df731 2024-11-17T22:41:51,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/537382436e5843ce8f5231b1019a156a is 50, key is test_row_0/B:col10/1731883310612/Put/seqid=0 2024-11-17T22:41:51,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742257_1433 (size=12001) 2024-11-17T22:41:51,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T22:41:51,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:51,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:51,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:51,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T22:41:51,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T22:41:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:51,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883371297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:51,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883371427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:51,430 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:51,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/537382436e5843ce8f5231b1019a156a 2024-11-17T22:41:51,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T22:41:51,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:51,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:51,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:51,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883371437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883371437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:51,439 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:51,439 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:51,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/14ef6377ed034f47bc7650bd42558a34 is 50, key is test_row_0/C:col10/1731883310612/Put/seqid=0 2024-11-17T22:41:51,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883371446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:51,448 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:51,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:51,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742258_1434 (size=12001) 2024-11-17T22:41:51,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T22:41:51,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:51,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:51,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:51,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T22:41:51,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T22:41:51,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:51,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:51,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:51,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:51,746 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/becf4610e8764765a8293f5a20013aca, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f8a349f209c649d392f7c5f3f528639f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/1a4e21eaf9fc47088f862504aaaa655a] to archive 2024-11-17T22:41:51,747 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:41:51,748 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/becf4610e8764765a8293f5a20013aca to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/becf4610e8764765a8293f5a20013aca 2024-11-17T22:41:51,749 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f8a349f209c649d392f7c5f3f528639f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f8a349f209c649d392f7c5f3f528639f 2024-11-17T22:41:51,750 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/1a4e21eaf9fc47088f862504aaaa655a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/1a4e21eaf9fc47088f862504aaaa655a 2024-11-17T22:41:51,751 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/ac84bc361d814a7498af2938252a3731, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/a9c8ec5aaa524691b25d5feafc1e6a52, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/164e337d229c4dee8842e6af5b2b1e5d] to archive 2024-11-17T22:41:51,751 WARN [regionserver/1a6e40b21a48:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 1 2024-11-17T22:41:51,751 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:41:51,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/ac84bc361d814a7498af2938252a3731 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/ac84bc361d814a7498af2938252a3731 2024-11-17T22:41:51,753 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/a9c8ec5aaa524691b25d5feafc1e6a52 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/a9c8ec5aaa524691b25d5feafc1e6a52 2024-11-17T22:41:51,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/164e337d229c4dee8842e6af5b2b1e5d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/164e337d229c4dee8842e6af5b2b1e5d 2024-11-17T22:41:51,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/00ceb06971164db3adb485bc33263836, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/0b548e805c3f43e58d9cbff00f8fbd3e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c967ed3ab1f844eabacc8bc4c9c82a96] to archive 2024-11-17T22:41:51,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:41:51,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/00ceb06971164db3adb485bc33263836 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/00ceb06971164db3adb485bc33263836 2024-11-17T22:41:51,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/0b548e805c3f43e58d9cbff00f8fbd3e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/0b548e805c3f43e58d9cbff00f8fbd3e 2024-11-17T22:41:51,759 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a6e40b21a48:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c967ed3ab1f844eabacc8bc4c9c82a96 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c967ed3ab1f844eabacc8bc4c9c82a96 2024-11-17T22:41:51,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883371816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:51,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/14ef6377ed034f47bc7650bd42558a34 2024-11-17T22:41:51,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/83dbed8c6ecb4f72aafddb33c57df731 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83dbed8c6ecb4f72aafddb33c57df731 2024-11-17T22:41:51,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83dbed8c6ecb4f72aafddb33c57df731, entries=200, sequenceid=76, filesize=14.0 K 2024-11-17T22:41:51,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/537382436e5843ce8f5231b1019a156a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/537382436e5843ce8f5231b1019a156a 2024-11-17T22:41:51,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/537382436e5843ce8f5231b1019a156a, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T22:41:51,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/14ef6377ed034f47bc7650bd42558a34 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/14ef6377ed034f47bc7650bd42558a34 2024-11-17T22:41:51,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/14ef6377ed034f47bc7650bd42558a34, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T22:41:51,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6c99de6ca45c57ddad88306e57b16183 in 1253ms, sequenceid=76, compaction requested=false 2024-11-17T22:41:51,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:51,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T22:41:51,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:51,897 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:41:51,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/2e81a38ddc8b4e9da6de6d91c8aa90ef is 50, key is test_row_0/A:col10/1731883310634/Put/seqid=0 2024-11-17T22:41:51,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742259_1435 (size=12001) 2024-11-17T22:41:52,306 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/2e81a38ddc8b4e9da6de6d91c8aa90ef 2024-11-17T22:41:52,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/b81b890a3c574362b66a52c6abe1f03f is 50, key is test_row_0/B:col10/1731883310634/Put/seqid=0 2024-11-17T22:41:52,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742260_1436 (size=12001) 2024-11-17T22:41:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T22:41:52,714 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/b81b890a3c574362b66a52c6abe1f03f 2024-11-17T22:41:52,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/d69600fb740248f99665c149abc0b00b is 50, key is test_row_0/C:col10/1731883310634/Put/seqid=0 2024-11-17T22:41:52,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742261_1437 (size=12001) 2024-11-17T22:41:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:52,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:52,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883372945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:53,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883373056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:53,136 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/d69600fb740248f99665c149abc0b00b 2024-11-17T22:41:53,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/2e81a38ddc8b4e9da6de6d91c8aa90ef as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2e81a38ddc8b4e9da6de6d91c8aa90ef 2024-11-17T22:41:53,142 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2e81a38ddc8b4e9da6de6d91c8aa90ef, entries=150, sequenceid=89, filesize=11.7 K 2024-11-17T22:41:53,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/b81b890a3c574362b66a52c6abe1f03f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/b81b890a3c574362b66a52c6abe1f03f 2024-11-17T22:41:53,146 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/b81b890a3c574362b66a52c6abe1f03f, entries=150, sequenceid=89, filesize=11.7 K 2024-11-17T22:41:53,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/d69600fb740248f99665c149abc0b00b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d69600fb740248f99665c149abc0b00b 2024-11-17T22:41:53,150 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d69600fb740248f99665c149abc0b00b, entries=150, sequenceid=89, filesize=11.7 K 2024-11-17T22:41:53,151 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6c99de6ca45c57ddad88306e57b16183 in 1254ms, sequenceid=89, compaction requested=true 2024-11-17T22:41:53,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:53,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:53,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-17T22:41:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-17T22:41:53,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-17T22:41:53,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6300 sec 2024-11-17T22:41:53,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.6360 sec 2024-11-17T22:41:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:53,275 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:53,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:41:53,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/dcdd8e44d6e7475e96af38b870057e81 is 50, key is test_row_0/A:col10/1731883312940/Put/seqid=0 2024-11-17T22:41:53,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742262_1438 (size=14341) 2024-11-17T22:41:53,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883373334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:53,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883373447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:53,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883373447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:53,449 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883373467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883373467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883373467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:53,474 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,474 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:53,475 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:53,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:53,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883373657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:53,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:53,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/dcdd8e44d6e7475e96af38b870057e81 2024-11-17T22:41:53,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/db631484d9564c5ab8aea6f04e3f4a60 is 50, key is test_row_0/B:col10/1731883312940/Put/seqid=0 2024-11-17T22:41:53,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742263_1439 (size=12001) 2024-11-17T22:41:53,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883373967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:54,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/db631484d9564c5ab8aea6f04e3f4a60 2024-11-17T22:41:54,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/55769458fd15436b80776abf1ff72bab is 50, key is test_row_0/C:col10/1731883312940/Put/seqid=0 2024-11-17T22:41:54,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742264_1440 (size=12001) 2024-11-17T22:41:54,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/55769458fd15436b80776abf1ff72bab 2024-11-17T22:41:54,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/dcdd8e44d6e7475e96af38b870057e81 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/dcdd8e44d6e7475e96af38b870057e81 2024-11-17T22:41:54,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/dcdd8e44d6e7475e96af38b870057e81, entries=200, sequenceid=114, filesize=14.0 K 2024-11-17T22:41:54,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/db631484d9564c5ab8aea6f04e3f4a60 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/db631484d9564c5ab8aea6f04e3f4a60 2024-11-17T22:41:54,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/db631484d9564c5ab8aea6f04e3f4a60, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T22:41:54,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/55769458fd15436b80776abf1ff72bab as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/55769458fd15436b80776abf1ff72bab 2024-11-17T22:41:54,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/55769458fd15436b80776abf1ff72bab, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T22:41:54,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6c99de6ca45c57ddad88306e57b16183 in 846ms, sequenceid=114, compaction requested=true 2024-11-17T22:41:54,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:54,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:54,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:54,121 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:41:54,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:54,121 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:41:54,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:54,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:54,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:54,124 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:41:54,124 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:41:54,124 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/A is initiating minor compaction (all files) 2024-11-17T22:41:54,124 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/B is initiating minor compaction (all files) 2024-11-17T22:41:54,124 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/B in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:54,124 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/A in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:54,124 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/614cdc28137a43c2add90f3cca6a94a9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83dbed8c6ecb4f72aafddb33c57df731, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2e81a38ddc8b4e9da6de6d91c8aa90ef, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/dcdd8e44d6e7475e96af38b870057e81] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=51.5 K 2024-11-17T22:41:54,124 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/031fae5be3e649bf8bcea7d6c356c081, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/537382436e5843ce8f5231b1019a156a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/b81b890a3c574362b66a52c6abe1f03f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/db631484d9564c5ab8aea6f04e3f4a60] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=47.0 K 2024-11-17T22:41:54,125 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 031fae5be3e649bf8bcea7d6c356c081, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883308584 2024-11-17T22:41:54,125 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 614cdc28137a43c2add90f3cca6a94a9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883308584 2024-11-17T22:41:54,125 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83dbed8c6ecb4f72aafddb33c57df731, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731883309271 2024-11-17T22:41:54,125 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 537382436e5843ce8f5231b1019a156a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731883309271 2024-11-17T22:41:54,125 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e81a38ddc8b4e9da6de6d91c8aa90ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731883310617 2024-11-17T22:41:54,125 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcdd8e44d6e7475e96af38b870057e81, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731883312901 2024-11-17T22:41:54,125 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b81b890a3c574362b66a52c6abe1f03f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731883310617 2024-11-17T22:41:54,126 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting db631484d9564c5ab8aea6f04e3f4a60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731883312940 2024-11-17T22:41:54,135 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#B#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:54,136 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/cf30ffa8b88142a39104d75a41368ced is 50, key is test_row_0/B:col10/1731883312940/Put/seqid=0 2024-11-17T22:41:54,144 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:54,144 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/d8c1c8c402d34acf91ec8e1f8f57a994 is 50, key is test_row_0/A:col10/1731883312940/Put/seqid=0 2024-11-17T22:41:54,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742265_1441 (size=12139) 2024-11-17T22:41:54,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742266_1442 (size=12139) 2024-11-17T22:41:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:54,491 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:54,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:41:54,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/eed4c993f69747c5a428a33cb6a2f1a2 is 50, key is test_row_0/A:col10/1731883313332/Put/seqid=0 2024-11-17T22:41:54,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742267_1443 (size=14341) 2024-11-17T22:41:54,550 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/cf30ffa8b88142a39104d75a41368ced as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/cf30ffa8b88142a39104d75a41368ced 2024-11-17T22:41:54,554 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/B of 6c99de6ca45c57ddad88306e57b16183 into cf30ffa8b88142a39104d75a41368ced(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:54,554 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:54,554 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/B, priority=12, startTime=1731883314121; duration=0sec 2024-11-17T22:41:54,554 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:54,554 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:B 2024-11-17T22:41:54,555 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:41:54,555 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/d8c1c8c402d34acf91ec8e1f8f57a994 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/d8c1c8c402d34acf91ec8e1f8f57a994 2024-11-17T22:41:54,555 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:41:54,555 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/C is initiating minor compaction (all files) 2024-11-17T22:41:54,555 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/C in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:54,555 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/04efb083e0d341d89001a87b747de180, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/14ef6377ed034f47bc7650bd42558a34, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d69600fb740248f99665c149abc0b00b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/55769458fd15436b80776abf1ff72bab] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=47.0 K 2024-11-17T22:41:54,556 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 04efb083e0d341d89001a87b747de180, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731883308584 2024-11-17T22:41:54,556 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 14ef6377ed034f47bc7650bd42558a34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731883309271 2024-11-17T22:41:54,556 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d69600fb740248f99665c149abc0b00b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731883310617 2024-11-17T22:41:54,557 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 55769458fd15436b80776abf1ff72bab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731883312940 2024-11-17T22:41:54,559 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/A of 6c99de6ca45c57ddad88306e57b16183 into d8c1c8c402d34acf91ec8e1f8f57a994(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:54,559 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:54,559 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/A, priority=12, startTime=1731883314121; duration=0sec 2024-11-17T22:41:54,559 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:54,559 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:A 2024-11-17T22:41:54,567 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#C#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:54,568 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/9b370b0099da47c6a5ffd5504d3fe8d8 is 50, key is test_row_0/C:col10/1731883312940/Put/seqid=0 2024-11-17T22:41:54,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742268_1444 (size=12139) 2024-11-17T22:41:54,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883374641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T22:41:54,677 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:54,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-17T22:41:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T22:41:54,679 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:54,680 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:54,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883374747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T22:41:54,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T22:41:54,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:54,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:54,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:54,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:54,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:54,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/eed4c993f69747c5a428a33cb6a2f1a2 2024-11-17T22:41:54,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/d36bb40d45724797adb188609d538fb0 is 50, key is test_row_0/B:col10/1731883313332/Put/seqid=0 2024-11-17T22:41:54,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742269_1445 (size=12001) 2024-11-17T22:41:54,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883374956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:54,976 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/9b370b0099da47c6a5ffd5504d3fe8d8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9b370b0099da47c6a5ffd5504d3fe8d8 2024-11-17T22:41:54,980 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/C of 6c99de6ca45c57ddad88306e57b16183 into 9b370b0099da47c6a5ffd5504d3fe8d8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:54,980 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:54,980 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/C, priority=12, startTime=1731883314121; duration=0sec 2024-11-17T22:41:54,981 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:54,981 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:C 2024-11-17T22:41:54,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T22:41:54,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:54,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:54,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:54,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:54,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T22:41:55,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T22:41:55,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:55,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:55,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:55,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883375277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:55,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T22:41:55,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:55,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:55,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:55,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T22:41:55,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/d36bb40d45724797adb188609d538fb0 2024-11-17T22:41:55,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/16ed9a4632ee4142b8570ebbf16ac3cc is 50, key is test_row_0/C:col10/1731883313332/Put/seqid=0 2024-11-17T22:41:55,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742270_1446 (size=12001) 2024-11-17T22:41:55,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T22:41:55,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:55,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:55,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:55,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T22:41:55,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:55,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:55,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:55,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:55,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/16ed9a4632ee4142b8570ebbf16ac3cc 2024-11-17T22:41:55,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/eed4c993f69747c5a428a33cb6a2f1a2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/eed4c993f69747c5a428a33cb6a2f1a2 2024-11-17T22:41:55,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/eed4c993f69747c5a428a33cb6a2f1a2, entries=200, sequenceid=126, filesize=14.0 K 2024-11-17T22:41:55,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/d36bb40d45724797adb188609d538fb0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d36bb40d45724797adb188609d538fb0 2024-11-17T22:41:55,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d36bb40d45724797adb188609d538fb0, entries=150, sequenceid=126, filesize=11.7 K 2024-11-17T22:41:55,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/16ed9a4632ee4142b8570ebbf16ac3cc as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/16ed9a4632ee4142b8570ebbf16ac3cc 2024-11-17T22:41:55,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/16ed9a4632ee4142b8570ebbf16ac3cc, entries=150, sequenceid=126, filesize=11.7 K 2024-11-17T22:41:55,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6c99de6ca45c57ddad88306e57b16183 in 1241ms, sequenceid=126, compaction requested=false 2024-11-17T22:41:55,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:55,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T22:41:55,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:55,747 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:41:55,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/83418ae4b105464696b0e5e177d64b24 is 50, key is test_row_0/A:col10/1731883314624/Put/seqid=0 2024-11-17T22:41:55,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742271_1447 (size=12151) 2024-11-17T22:41:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:55,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T22:41:55,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883375846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:55,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883375957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:56,154 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/83418ae4b105464696b0e5e177d64b24 2024-11-17T22:41:56,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/28293dfc84b644e18f70e2c2893cc809 is 50, key is test_row_0/B:col10/1731883314624/Put/seqid=0 2024-11-17T22:41:56,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742272_1448 (size=12151) 2024-11-17T22:41:56,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883376167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883376477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:56,566 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/28293dfc84b644e18f70e2c2893cc809 2024-11-17T22:41:56,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/c37c116d141f4b6b92578b5a01a5a8dd is 50, key is test_row_0/C:col10/1731883314624/Put/seqid=0 2024-11-17T22:41:56,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742273_1449 (size=12151) 2024-11-17T22:41:56,575 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/c37c116d141f4b6b92578b5a01a5a8dd 2024-11-17T22:41:56,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/83418ae4b105464696b0e5e177d64b24 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83418ae4b105464696b0e5e177d64b24 2024-11-17T22:41:56,581 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83418ae4b105464696b0e5e177d64b24, entries=150, sequenceid=153, filesize=11.9 K 2024-11-17T22:41:56,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/28293dfc84b644e18f70e2c2893cc809 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/28293dfc84b644e18f70e2c2893cc809 2024-11-17T22:41:56,585 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/28293dfc84b644e18f70e2c2893cc809, entries=150, sequenceid=153, filesize=11.9 K 2024-11-17T22:41:56,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/c37c116d141f4b6b92578b5a01a5a8dd as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c37c116d141f4b6b92578b5a01a5a8dd 2024-11-17T22:41:56,589 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c37c116d141f4b6b92578b5a01a5a8dd, entries=150, sequenceid=153, filesize=11.9 K 2024-11-17T22:41:56,589 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6c99de6ca45c57ddad88306e57b16183 in 842ms, sequenceid=153, compaction requested=true 2024-11-17T22:41:56,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:56,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:56,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-17T22:41:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-17T22:41:56,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-17T22:41:56,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9100 sec 2024-11-17T22:41:56,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.9140 sec 2024-11-17T22:41:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T22:41:56,827 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:56,827 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-17T22:41:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T22:41:56,829 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:56,830 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:56,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T22:41:56,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T22:41:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:56,982 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:41:56,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/126452f9b58143fb944d5b5afa866a5d is 50, key is test_row_0/A:col10/1731883315845/Put/seqid=0 2024-11-17T22:41:56,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742274_1450 (size=12151) 2024-11-17T22:41:56,990 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/126452f9b58143fb944d5b5afa866a5d 2024-11-17T22:41:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:56,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:56,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/67db2355f0f34d73a1ac9065b8fdc278 is 50, key is test_row_0/B:col10/1731883315845/Put/seqid=0 2024-11-17T22:41:57,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742275_1451 (size=12151) 2024-11-17T22:41:57,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883377113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T22:41:57,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883377227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:57,401 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/67db2355f0f34d73a1ac9065b8fdc278 2024-11-17T22:41:57,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/691968656b9a458daf327d4b9a859b3e is 50, key is test_row_0/C:col10/1731883315845/Put/seqid=0 2024-11-17T22:41:57,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742276_1452 (size=12151) 2024-11-17T22:41:57,409 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/691968656b9a458daf327d4b9a859b3e 2024-11-17T22:41:57,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/126452f9b58143fb944d5b5afa866a5d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/126452f9b58143fb944d5b5afa866a5d 2024-11-17T22:41:57,416 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/126452f9b58143fb944d5b5afa866a5d, entries=150, sequenceid=165, filesize=11.9 K 2024-11-17T22:41:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/67db2355f0f34d73a1ac9065b8fdc278 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/67db2355f0f34d73a1ac9065b8fdc278 2024-11-17T22:41:57,419 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/67db2355f0f34d73a1ac9065b8fdc278, entries=150, sequenceid=165, filesize=11.9 K 2024-11-17T22:41:57,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/691968656b9a458daf327d4b9a859b3e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/691968656b9a458daf327d4b9a859b3e 2024-11-17T22:41:57,426 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/691968656b9a458daf327d4b9a859b3e, entries=150, sequenceid=165, filesize=11.9 K 2024-11-17T22:41:57,427 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6c99de6ca45c57ddad88306e57b16183 in 445ms, sequenceid=165, compaction requested=true 2024-11-17T22:41:57,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:57,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:57,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-17T22:41:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-17T22:41:57,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-17T22:41:57,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 598 msec 2024-11-17T22:41:57,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 603 msec 2024-11-17T22:41:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:57,444 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:57,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:41:57,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/8b884cf1d6f0408abcc2c19a6ccf9f0d is 50, key is test_row_0/A:col10/1731883317443/Put/seqid=0 2024-11-17T22:41:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T22:41:57,457 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:41:57,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:41:57,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742277_1453 (size=14541) 2024-11-17T22:41:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-17T22:41:57,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/8b884cf1d6f0408abcc2c19a6ccf9f0d 2024-11-17T22:41:57,459 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:41:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T22:41:57,460 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:41:57,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:41:57,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/09fc08c533dd4695987ce76102ab9996 is 50, key is test_row_0/B:col10/1731883317443/Put/seqid=0 2024-11-17T22:41:57,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742278_1454 (size=12151) 2024-11-17T22:41:57,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57244 deadline: 1731883377487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:57,495 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:57,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883377491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:57,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883377492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:57,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57216 deadline: 1731883377497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:57,507 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:57,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57222 deadline: 1731883377516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:57,523 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_0' in 6c99de6ca45c57ddad88306e57b16183 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:41:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T22:41:57,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T22:41:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:57,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:57,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:57,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883377606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:57,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883377616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:57,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T22:41:57,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:57,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:57,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:57,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:57,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T22:41:57,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883377817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:57,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883377836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:57,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/09fc08c533dd4695987ce76102ab9996 2024-11-17T22:41:57,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/9fa193d0674d46b28f0870c7c572481f is 50, key is test_row_0/C:col10/1731883317443/Put/seqid=0 2024-11-17T22:41:57,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742279_1455 (size=12151) 2024-11-17T22:41:57,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T22:41:57,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:57,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:57,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:57,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:57,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:58,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T22:41:58,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:58,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:58,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:58,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:58,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T22:41:58,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883378137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:58,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883378157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:58,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T22:41:58,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:58,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:58,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:58,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:58,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:41:58,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/9fa193d0674d46b28f0870c7c572481f 2024-11-17T22:41:58,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/8b884cf1d6f0408abcc2c19a6ccf9f0d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/8b884cf1d6f0408abcc2c19a6ccf9f0d 2024-11-17T22:41:58,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/8b884cf1d6f0408abcc2c19a6ccf9f0d, entries=200, sequenceid=190, filesize=14.2 K 2024-11-17T22:41:58,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/09fc08c533dd4695987ce76102ab9996 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/09fc08c533dd4695987ce76102ab9996 2024-11-17T22:41:58,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/09fc08c533dd4695987ce76102ab9996, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T22:41:58,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/9fa193d0674d46b28f0870c7c572481f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9fa193d0674d46b28f0870c7c572481f 2024-11-17T22:41:58,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9fa193d0674d46b28f0870c7c572481f, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T22:41:58,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6c99de6ca45c57ddad88306e57b16183 in 850ms, sequenceid=190, compaction requested=true 2024-11-17T22:41:58,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:58,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:41:58,294 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T22:41:58,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:58,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:41:58,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:58,294 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T22:41:58,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:41:58,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:58,295 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 65323 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T22:41:58,295 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60593 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T22:41:58,295 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/A is initiating minor compaction (all files) 2024-11-17T22:41:58,295 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/B is initiating minor compaction (all files) 2024-11-17T22:41:58,295 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/A in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:58,295 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/B in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:58,295 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/d8c1c8c402d34acf91ec8e1f8f57a994, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/eed4c993f69747c5a428a33cb6a2f1a2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83418ae4b105464696b0e5e177d64b24, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/126452f9b58143fb944d5b5afa866a5d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/8b884cf1d6f0408abcc2c19a6ccf9f0d] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=63.8 K 2024-11-17T22:41:58,295 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/cf30ffa8b88142a39104d75a41368ced, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d36bb40d45724797adb188609d538fb0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/28293dfc84b644e18f70e2c2893cc809, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/67db2355f0f34d73a1ac9065b8fdc278, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/09fc08c533dd4695987ce76102ab9996] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=59.2 K 2024-11-17T22:41:58,296 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8c1c8c402d34acf91ec8e1f8f57a994, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731883312940 2024-11-17T22:41:58,296 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting cf30ffa8b88142a39104d75a41368ced, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731883312940 2024-11-17T22:41:58,296 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d36bb40d45724797adb188609d538fb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731883313304 2024-11-17T22:41:58,296 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed4c993f69747c5a428a33cb6a2f1a2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731883313304 2024-11-17T22:41:58,296 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83418ae4b105464696b0e5e177d64b24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731883314624 2024-11-17T22:41:58,296 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 28293dfc84b644e18f70e2c2893cc809, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731883314624 2024-11-17T22:41:58,296 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 126452f9b58143fb944d5b5afa866a5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731883315817 2024-11-17T22:41:58,296 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 67db2355f0f34d73a1ac9065b8fdc278, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731883315817 2024-11-17T22:41:58,296 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b884cf1d6f0408abcc2c19a6ccf9f0d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731883317092 2024-11-17T22:41:58,297 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 09fc08c533dd4695987ce76102ab9996, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731883317105 2024-11-17T22:41:58,304 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#B#compaction#389 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:58,305 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:58,305 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/c797f91f60284a02ab72740499b95468 is 50, key is test_row_0/A:col10/1731883317443/Put/seqid=0 2024-11-17T22:41:58,305 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/b5b25b9e2fed47c189e7bb9a54050e12 is 50, key is test_row_0/B:col10/1731883317443/Put/seqid=0 2024-11-17T22:41:58,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742280_1456 (size=12459) 2024-11-17T22:41:58,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742281_1457 (size=12459) 2024-11-17T22:41:58,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T22:41:58,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:58,376 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:41:58,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/9c13adb52e00489090c1ddecde733745 is 50, key is test_row_0/A:col10/1731883317478/Put/seqid=0 2024-11-17T22:41:58,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742282_1458 (size=9757) 2024-11-17T22:41:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T22:41:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:41:58,720 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/b5b25b9e2fed47c189e7bb9a54050e12 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/b5b25b9e2fed47c189e7bb9a54050e12 2024-11-17T22:41:58,720 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/c797f91f60284a02ab72740499b95468 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/c797f91f60284a02ab72740499b95468 2024-11-17T22:41:58,724 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/A of 6c99de6ca45c57ddad88306e57b16183 into c797f91f60284a02ab72740499b95468(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:58,724 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:58,724 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/A, priority=11, startTime=1731883318294; duration=0sec 2024-11-17T22:41:58,724 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:41:58,724 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:A 2024-11-17T22:41:58,724 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/B of 6c99de6ca45c57ddad88306e57b16183 into b5b25b9e2fed47c189e7bb9a54050e12(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:58,724 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T22:41:58,724 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:58,724 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/B, priority=11, startTime=1731883318294; duration=0sec 2024-11-17T22:41:58,724 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:58,724 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:B 2024-11-17T22:41:58,725 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60593 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T22:41:58,725 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/C is initiating minor compaction (all files) 2024-11-17T22:41:58,725 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/C in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:58,725 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9b370b0099da47c6a5ffd5504d3fe8d8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/16ed9a4632ee4142b8570ebbf16ac3cc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c37c116d141f4b6b92578b5a01a5a8dd, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/691968656b9a458daf327d4b9a859b3e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9fa193d0674d46b28f0870c7c572481f] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=59.2 K 2024-11-17T22:41:58,726 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b370b0099da47c6a5ffd5504d3fe8d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731883312940 2024-11-17T22:41:58,726 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16ed9a4632ee4142b8570ebbf16ac3cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731883313304 2024-11-17T22:41:58,726 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting c37c116d141f4b6b92578b5a01a5a8dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731883314624 2024-11-17T22:41:58,726 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 691968656b9a458daf327d4b9a859b3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731883315817 2024-11-17T22:41:58,727 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fa193d0674d46b28f0870c7c572481f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731883317105 2024-11-17T22:41:58,733 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:41:58,738 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/230b73fb90a946c3b54ee39d7a5f511f is 50, key is test_row_0/C:col10/1731883317443/Put/seqid=0 2024-11-17T22:41:58,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742283_1459 (size=12459) 2024-11-17T22:41:58,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883378764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:58,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883378769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:58,784 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/9c13adb52e00489090c1ddecde733745 2024-11-17T22:41:58,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/95013405c31c4667b7a1c994ea49d257 is 50, key is test_row_0/B:col10/1731883317478/Put/seqid=0 2024-11-17T22:41:58,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742284_1460 (size=9757) 2024-11-17T22:41:58,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883378877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:58,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883378877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:59,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883379086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:59,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:59,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883379096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:59,146 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/230b73fb90a946c3b54ee39d7a5f511f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/230b73fb90a946c3b54ee39d7a5f511f 2024-11-17T22:41:59,150 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/C of 6c99de6ca45c57ddad88306e57b16183 into 230b73fb90a946c3b54ee39d7a5f511f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:41:59,150 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:59,150 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/C, priority=11, startTime=1731883318294; duration=0sec 2024-11-17T22:41:59,150 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:41:59,150 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:C 2024-11-17T22:41:59,199 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/95013405c31c4667b7a1c994ea49d257 2024-11-17T22:41:59,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/e0aae51e743241508dd540eda3d4b474 is 50, key is test_row_0/C:col10/1731883317478/Put/seqid=0 2024-11-17T22:41:59,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742285_1461 (size=9757) 2024-11-17T22:41:59,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883379397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:59,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883379407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T22:41:59,609 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/e0aae51e743241508dd540eda3d4b474 2024-11-17T22:41:59,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/9c13adb52e00489090c1ddecde733745 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/9c13adb52e00489090c1ddecde733745 2024-11-17T22:41:59,616 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/9c13adb52e00489090c1ddecde733745, entries=100, sequenceid=201, filesize=9.5 K 2024-11-17T22:41:59,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/95013405c31c4667b7a1c994ea49d257 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/95013405c31c4667b7a1c994ea49d257 2024-11-17T22:41:59,620 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/95013405c31c4667b7a1c994ea49d257, entries=100, sequenceid=201, filesize=9.5 K 2024-11-17T22:41:59,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/e0aae51e743241508dd540eda3d4b474 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/e0aae51e743241508dd540eda3d4b474 2024-11-17T22:41:59,623 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/e0aae51e743241508dd540eda3d4b474, entries=100, sequenceid=201, filesize=9.5 K 2024-11-17T22:41:59,624 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6c99de6ca45c57ddad88306e57b16183 in 1248ms, sequenceid=201, compaction requested=false 2024-11-17T22:41:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:41:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:41:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-17T22:41:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-17T22:41:59,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-17T22:41:59,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1650 sec 2024-11-17T22:41:59,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.1690 sec 2024-11-17T22:41:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:41:59,924 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:41:59,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T22:41:59,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/f57aaa2b5fee49ceb7f512cb2930dbcc is 50, key is test_row_0/A:col10/1731883318767/Put/seqid=0 2024-11-17T22:41:59,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742286_1462 (size=14541) 2024-11-17T22:41:59,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883379946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:41:59,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:41:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883379949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:41:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:41:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:00,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883380056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:00,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:00,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883380067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:00,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:00,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883380267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:00,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883380276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:00,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:00,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/f57aaa2b5fee49ceb7f512cb2930dbcc 2024-11-17T22:42:00,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/9df8485bf72b43a5b4ac35ceec3ad6d6 is 50, key is test_row_0/B:col10/1731883318767/Put/seqid=0 2024-11-17T22:42:00,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742287_1463 (size=12151) 2024-11-17T22:42:00,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883380577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:00,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883380587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:00,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:00,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:00,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/9df8485bf72b43a5b4ac35ceec3ad6d6 2024-11-17T22:42:00,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/92467af6b9134c81a7582a451affa476 is 50, key is test_row_0/C:col10/1731883318767/Put/seqid=0 2024-11-17T22:42:00,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742288_1464 (size=12151) 2024-11-17T22:42:00,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/92467af6b9134c81a7582a451affa476 2024-11-17T22:42:00,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/f57aaa2b5fee49ceb7f512cb2930dbcc as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f57aaa2b5fee49ceb7f512cb2930dbcc 2024-11-17T22:42:00,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f57aaa2b5fee49ceb7f512cb2930dbcc, entries=200, sequenceid=230, filesize=14.2 K 2024-11-17T22:42:00,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/9df8485bf72b43a5b4ac35ceec3ad6d6 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/9df8485bf72b43a5b4ac35ceec3ad6d6 2024-11-17T22:42:00,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/9df8485bf72b43a5b4ac35ceec3ad6d6, entries=150, sequenceid=230, filesize=11.9 K 2024-11-17T22:42:00,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/92467af6b9134c81a7582a451affa476 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/92467af6b9134c81a7582a451affa476 2024-11-17T22:42:00,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/92467af6b9134c81a7582a451affa476, entries=150, sequenceid=230, filesize=11.9 K 2024-11-17T22:42:00,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6c99de6ca45c57ddad88306e57b16183 in 846ms, sequenceid=230, compaction requested=true 2024-11-17T22:42:00,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:00,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:42:00,770 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:00,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:00,771 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:00,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:42:00,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:00,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:00,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:00,771 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36757 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:00,771 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/A is initiating minor compaction (all files) 2024-11-17T22:42:00,772 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/A in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:00,772 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:00,772 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/B is initiating minor compaction (all files) 2024-11-17T22:42:00,772 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/B in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:00,772 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/c797f91f60284a02ab72740499b95468, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/9c13adb52e00489090c1ddecde733745, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f57aaa2b5fee49ceb7f512cb2930dbcc] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=35.9 K 2024-11-17T22:42:00,772 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/b5b25b9e2fed47c189e7bb9a54050e12, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/95013405c31c4667b7a1c994ea49d257, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/9df8485bf72b43a5b4ac35ceec3ad6d6] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=33.6 K 2024-11-17T22:42:00,772 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting c797f91f60284a02ab72740499b95468, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731883317105 2024-11-17T22:42:00,772 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b25b9e2fed47c189e7bb9a54050e12, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731883317105 2024-11-17T22:42:00,772 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c13adb52e00489090c1ddecde733745, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731883317478 2024-11-17T22:42:00,772 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 95013405c31c4667b7a1c994ea49d257, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731883317478 2024-11-17T22:42:00,772 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting f57aaa2b5fee49ceb7f512cb2930dbcc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731883318757 2024-11-17T22:42:00,772 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 9df8485bf72b43a5b4ac35ceec3ad6d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731883318763 2024-11-17T22:42:00,779 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:00,780 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#A#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:00,780 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/557e520893dc406689600e6d6c28983a is 50, key is test_row_0/B:col10/1731883318767/Put/seqid=0 2024-11-17T22:42:00,780 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/f8386ee4714a40d7b652965917e2ce33 is 50, key is test_row_0/A:col10/1731883318767/Put/seqid=0 2024-11-17T22:42:00,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742289_1465 (size=12561) 2024-11-17T22:42:00,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742290_1466 (size=12561) 2024-11-17T22:42:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:42:01,091 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:01,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:42:01,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/ae90b111b90441d3bd1376ed5ef99e88 is 50, key is test_row_0/A:col10/1731883321089/Put/seqid=0 2024-11-17T22:42:01,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742291_1467 (size=14541) 2024-11-17T22:42:01,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883381173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:01,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883381174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:01,189 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/f8386ee4714a40d7b652965917e2ce33 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f8386ee4714a40d7b652965917e2ce33 2024-11-17T22:42:01,189 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/557e520893dc406689600e6d6c28983a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/557e520893dc406689600e6d6c28983a 2024-11-17T22:42:01,196 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/B of 6c99de6ca45c57ddad88306e57b16183 into 557e520893dc406689600e6d6c28983a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:01,196 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/A of 6c99de6ca45c57ddad88306e57b16183 into f8386ee4714a40d7b652965917e2ce33(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:01,196 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:01,196 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:01,196 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/A, priority=13, startTime=1731883320770; duration=0sec 2024-11-17T22:42:01,196 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/B, priority=13, startTime=1731883320771; duration=0sec 2024-11-17T22:42:01,196 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:01,196 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:A 2024-11-17T22:42:01,196 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:01,196 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:B 2024-11-17T22:42:01,196 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:01,197 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:01,197 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/C is initiating minor compaction (all files) 2024-11-17T22:42:01,197 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/C in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:01,197 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/230b73fb90a946c3b54ee39d7a5f511f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/e0aae51e743241508dd540eda3d4b474, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/92467af6b9134c81a7582a451affa476] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=33.6 K 2024-11-17T22:42:01,198 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 230b73fb90a946c3b54ee39d7a5f511f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731883317105 2024-11-17T22:42:01,198 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0aae51e743241508dd540eda3d4b474, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731883317478 2024-11-17T22:42:01,198 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92467af6b9134c81a7582a451affa476, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731883318763 2024-11-17T22:42:01,203 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:01,203 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/5c4b269e20ea429eb9e72855c8abb279 is 50, key is test_row_0/C:col10/1731883318767/Put/seqid=0 2024-11-17T22:42:01,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742292_1468 (size=12561) 2024-11-17T22:42:01,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883381286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883381286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:01,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/ae90b111b90441d3bd1376ed5ef99e88 2024-11-17T22:42:01,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/fbc32884c6264fca9e234b420d26253b is 50, key is test_row_0/B:col10/1731883321089/Put/seqid=0 2024-11-17T22:42:01,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883381497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:01,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883381497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:01,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:01,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:01,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742293_1469 (size=12151) 2024-11-17T22:42:01,611 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/5c4b269e20ea429eb9e72855c8abb279 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/5c4b269e20ea429eb9e72855c8abb279 2024-11-17T22:42:01,614 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/C of 6c99de6ca45c57ddad88306e57b16183 into 5c4b269e20ea429eb9e72855c8abb279(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:01,614 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:01,614 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/C, priority=13, startTime=1731883320771; duration=0sec 2024-11-17T22:42:01,615 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:01,615 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:C 2024-11-17T22:42:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T22:42:01,628 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:01,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:42:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-17T22:42:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T22:42:01,630 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:42:01,630 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:42:01,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:42:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T22:42:01,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T22:42:01,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:01,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:01,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:01,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:01,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:01,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883381807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:01,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:01,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883381807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:01,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/fbc32884c6264fca9e234b420d26253b 2024-11-17T22:42:01,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/22059bc6b58a4b1c9b0d9e6ce56734a0 is 50, key is test_row_0/C:col10/1731883321089/Put/seqid=0 2024-11-17T22:42:01,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742294_1470 (size=12151) 2024-11-17T22:42:01,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T22:42:01,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:01,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:01,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:01,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T22:42:02,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T22:42:02,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:02,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T22:42:02,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:02,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T22:42:02,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/22059bc6b58a4b1c9b0d9e6ce56734a0 2024-11-17T22:42:02,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/ae90b111b90441d3bd1376ed5ef99e88 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ae90b111b90441d3bd1376ed5ef99e88 2024-11-17T22:42:02,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ae90b111b90441d3bd1376ed5ef99e88, entries=200, sequenceid=241, filesize=14.2 K 2024-11-17T22:42:02,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/fbc32884c6264fca9e234b420d26253b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/fbc32884c6264fca9e234b420d26253b 2024-11-17T22:42:02,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/fbc32884c6264fca9e234b420d26253b, entries=150, sequenceid=241, filesize=11.9 K 2024-11-17T22:42:02,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/22059bc6b58a4b1c9b0d9e6ce56734a0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/22059bc6b58a4b1c9b0d9e6ce56734a0 2024-11-17T22:42:02,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/22059bc6b58a4b1c9b0d9e6ce56734a0, entries=150, sequenceid=241, filesize=11.9 K 2024-11-17T22:42:02,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6c99de6ca45c57ddad88306e57b16183 in 1237ms, sequenceid=241, compaction requested=false 2024-11-17T22:42:02,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:42:02,331 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:02,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T22:42:02,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/24e3deec1f9d470999bcc00c29565ba1 is 50, key is test_row_0/A:col10/1731883321173/Put/seqid=0 2024-11-17T22:42:02,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742295_1471 (size=14741) 2024-11-17T22:42:02,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883382352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:02,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883382356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:02,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T22:42:02,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:02,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883382467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883382467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:02,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T22:42:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883382676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:02,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883382676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:02,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T22:42:02,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/24e3deec1f9d470999bcc00c29565ba1 2024-11-17T22:42:02,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/e0995f062bb643e7933ead9a83ea1ef0 is 50, key is test_row_0/B:col10/1731883321173/Put/seqid=0 2024-11-17T22:42:02,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742296_1472 (size=12301) 2024-11-17T22:42:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T22:42:02,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T22:42:02,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:02,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:02,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:02,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883382987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:02,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883382987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:03,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T22:42:03,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:03,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:03,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:03,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:03,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:03,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/e0995f062bb643e7933ead9a83ea1ef0 2024-11-17T22:42:03,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/d0a7a6d426f2490fb88c9c619c23b714 is 50, key is test_row_0/C:col10/1731883321173/Put/seqid=0 2024-11-17T22:42:03,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T22:42:03,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:03,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:03,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:03,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:03,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:03,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742297_1473 (size=12301) 2024-11-17T22:42:03,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/d0a7a6d426f2490fb88c9c619c23b714 2024-11-17T22:42:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/24e3deec1f9d470999bcc00c29565ba1 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/24e3deec1f9d470999bcc00c29565ba1 2024-11-17T22:42:03,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/24e3deec1f9d470999bcc00c29565ba1, entries=200, sequenceid=270, filesize=14.4 K 2024-11-17T22:42:03,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/e0995f062bb643e7933ead9a83ea1ef0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/e0995f062bb643e7933ead9a83ea1ef0 2024-11-17T22:42:03,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/e0995f062bb643e7933ead9a83ea1ef0, entries=150, sequenceid=270, filesize=12.0 K 2024-11-17T22:42:03,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/d0a7a6d426f2490fb88c9c619c23b714 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d0a7a6d426f2490fb88c9c619c23b714 2024-11-17T22:42:03,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d0a7a6d426f2490fb88c9c619c23b714, entries=150, sequenceid=270, filesize=12.0 K 2024-11-17T22:42:03,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6c99de6ca45c57ddad88306e57b16183 in 844ms, sequenceid=270, compaction requested=true 2024-11-17T22:42:03,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:03,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:42:03,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:03,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:42:03,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:03,176 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:03,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:03,176 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:03,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:03,176 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:03,176 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:03,176 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/A is initiating minor compaction (all files) 2024-11-17T22:42:03,176 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/B is initiating minor compaction (all files) 2024-11-17T22:42:03,177 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/A in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:03,177 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/B in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:03,177 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f8386ee4714a40d7b652965917e2ce33, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ae90b111b90441d3bd1376ed5ef99e88, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/24e3deec1f9d470999bcc00c29565ba1] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=40.9 K 2024-11-17T22:42:03,177 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/557e520893dc406689600e6d6c28983a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/fbc32884c6264fca9e234b420d26253b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/e0995f062bb643e7933ead9a83ea1ef0] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=36.1 K 2024-11-17T22:42:03,177 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8386ee4714a40d7b652965917e2ce33, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731883318763 2024-11-17T22:42:03,177 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 557e520893dc406689600e6d6c28983a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731883318763 2024-11-17T22:42:03,177 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting fbc32884c6264fca9e234b420d26253b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731883319925 2024-11-17T22:42:03,177 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae90b111b90441d3bd1376ed5ef99e88, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731883319925 2024-11-17T22:42:03,177 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e3deec1f9d470999bcc00c29565ba1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731883321149 2024-11-17T22:42:03,177 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting e0995f062bb643e7933ead9a83ea1ef0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731883321149 2024-11-17T22:42:03,183 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#B#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:03,184 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/3d43113de1b447bba17b4fdeebc1981f is 50, key is test_row_0/B:col10/1731883321173/Put/seqid=0 2024-11-17T22:42:03,186 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:03,186 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/1bd67d5a96d947c6b92f45d1650e45c8 is 50, key is test_row_0/A:col10/1731883321173/Put/seqid=0 2024-11-17T22:42:03,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742298_1474 (size=12813) 2024-11-17T22:42:03,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742299_1475 (size=12813) 2024-11-17T22:42:03,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T22:42:03,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:03,312 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T22:42:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/2f278b9be3ea4db494797ea0af3d20b0 is 50, key is test_row_0/A:col10/1731883322354/Put/seqid=0 2024-11-17T22:42:03,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742300_1476 (size=9857) 2024-11-17T22:42:03,319 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/2f278b9be3ea4db494797ea0af3d20b0 2024-11-17T22:42:03,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/5b5fa9753947414d8350f5a64d792991 is 50, key is test_row_0/B:col10/1731883322354/Put/seqid=0 2024-11-17T22:42:03,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742301_1477 (size=9857) 2024-11-17T22:42:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:42:03,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:03,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883383577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:03,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883383579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:03,601 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/1bd67d5a96d947c6b92f45d1650e45c8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/1bd67d5a96d947c6b92f45d1650e45c8 2024-11-17T22:42:03,601 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/3d43113de1b447bba17b4fdeebc1981f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/3d43113de1b447bba17b4fdeebc1981f 2024-11-17T22:42:03,604 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/A of 6c99de6ca45c57ddad88306e57b16183 into 1bd67d5a96d947c6b92f45d1650e45c8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:03,604 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/B of 6c99de6ca45c57ddad88306e57b16183 into 3d43113de1b447bba17b4fdeebc1981f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:03,604 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:03,604 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:03,604 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/A, priority=13, startTime=1731883323176; duration=0sec 2024-11-17T22:42:03,604 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/B, priority=13, startTime=1731883323176; duration=0sec 2024-11-17T22:42:03,605 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:03,605 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:A 2024-11-17T22:42:03,605 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:03,605 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:B 2024-11-17T22:42:03,605 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:03,605 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:03,605 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/C is initiating minor compaction (all files) 2024-11-17T22:42:03,605 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/C in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:03,605 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/5c4b269e20ea429eb9e72855c8abb279, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/22059bc6b58a4b1c9b0d9e6ce56734a0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d0a7a6d426f2490fb88c9c619c23b714] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=36.1 K 2024-11-17T22:42:03,606 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4b269e20ea429eb9e72855c8abb279, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731883318763 2024-11-17T22:42:03,606 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22059bc6b58a4b1c9b0d9e6ce56734a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731883319925 2024-11-17T22:42:03,606 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0a7a6d426f2490fb88c9c619c23b714, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731883321149 2024-11-17T22:42:03,611 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:03,611 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/6b82f50b4f8740f99048f28d4b6db457 is 50, key is test_row_0/C:col10/1731883321173/Put/seqid=0 2024-11-17T22:42:03,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742302_1478 (size=12813) 2024-11-17T22:42:03,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883383687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:03,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883383697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:03,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:03,727 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/5b5fa9753947414d8350f5a64d792991 2024-11-17T22:42:03,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/93cf01355f50415e82666b62896a162a is 50, key is test_row_0/C:col10/1731883322354/Put/seqid=0 2024-11-17T22:42:03,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742303_1479 (size=9857) 2024-11-17T22:42:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T22:42:03,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883383897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:03,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883383907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:04,018 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/6b82f50b4f8740f99048f28d4b6db457 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/6b82f50b4f8740f99048f28d4b6db457 2024-11-17T22:42:04,021 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/C of 6c99de6ca45c57ddad88306e57b16183 into 6b82f50b4f8740f99048f28d4b6db457(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:04,021 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:04,021 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/C, priority=13, startTime=1731883323176; duration=0sec 2024-11-17T22:42:04,021 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:04,021 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:C 2024-11-17T22:42:04,136 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/93cf01355f50415e82666b62896a162a 2024-11-17T22:42:04,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/2f278b9be3ea4db494797ea0af3d20b0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2f278b9be3ea4db494797ea0af3d20b0 2024-11-17T22:42:04,142 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2f278b9be3ea4db494797ea0af3d20b0, entries=100, sequenceid=280, filesize=9.6 K 2024-11-17T22:42:04,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/5b5fa9753947414d8350f5a64d792991 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/5b5fa9753947414d8350f5a64d792991 2024-11-17T22:42:04,146 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/5b5fa9753947414d8350f5a64d792991, entries=100, sequenceid=280, filesize=9.6 K 2024-11-17T22:42:04,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/93cf01355f50415e82666b62896a162a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/93cf01355f50415e82666b62896a162a 2024-11-17T22:42:04,150 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/93cf01355f50415e82666b62896a162a, entries=100, sequenceid=280, filesize=9.6 K 2024-11-17T22:42:04,151 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 6c99de6ca45c57ddad88306e57b16183 in 839ms, sequenceid=280, compaction requested=false 2024-11-17T22:42:04,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:04,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:04,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-17T22:42:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-17T22:42:04,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-17T22:42:04,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5230 sec 2024-11-17T22:42:04,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.5270 sec 2024-11-17T22:42:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:42:04,212 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:04,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-17T22:42:04,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/ee38c76653d540cc8087890302246a4b is 50, key is test_row_0/A:col10/1731883324211/Put/seqid=0 2024-11-17T22:42:04,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742304_1480 (size=14741) 2024-11-17T22:42:04,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883384233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:04,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883384233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:04,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883384337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883384337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:04,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883384557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:04,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883384557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:04,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/ee38c76653d540cc8087890302246a4b 2024-11-17T22:42:04,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/d91173d93f214deeaba820a0eea547a0 is 50, key is test_row_0/B:col10/1731883324211/Put/seqid=0 2024-11-17T22:42:04,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742305_1481 (size=12301) 2024-11-17T22:42:04,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883384866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883384867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:05,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/d91173d93f214deeaba820a0eea547a0 2024-11-17T22:42:05,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/9a1959f6e1ae46848840561e697b042c is 50, key is test_row_0/C:col10/1731883324211/Put/seqid=0 2024-11-17T22:42:05,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742306_1482 (size=12301) 2024-11-17T22:42:05,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57196 deadline: 1731883385377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:05,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57210 deadline: 1731883385377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 , the old value is region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., hostname=1a6e40b21a48,43011,1731883188895, seqNum=2 because the exception is null or not the one we care about 2024-11-17T22:42:05,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/9a1959f6e1ae46848840561e697b042c 2024-11-17T22:42:05,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/ee38c76653d540cc8087890302246a4b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ee38c76653d540cc8087890302246a4b 2024-11-17T22:42:05,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ee38c76653d540cc8087890302246a4b, entries=200, sequenceid=310, filesize=14.4 K 2024-11-17T22:42:05,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/d91173d93f214deeaba820a0eea547a0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d91173d93f214deeaba820a0eea547a0 2024-11-17T22:42:05,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d91173d93f214deeaba820a0eea547a0, entries=150, sequenceid=310, filesize=12.0 K 2024-11-17T22:42:05,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/9a1959f6e1ae46848840561e697b042c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9a1959f6e1ae46848840561e697b042c 2024-11-17T22:42:05,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9a1959f6e1ae46848840561e697b042c, entries=150, sequenceid=310, filesize=12.0 K 2024-11-17T22:42:05,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 6c99de6ca45c57ddad88306e57b16183 in 1246ms, sequenceid=310, compaction requested=true 2024-11-17T22:42:05,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:05,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:42:05,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:05,458 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:05,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:42:05,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:05,458 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:05,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c99de6ca45c57ddad88306e57b16183:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:05,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:05,459 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37411 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:05,459 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:05,459 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/A is initiating minor compaction (all files) 2024-11-17T22:42:05,459 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/B is initiating minor compaction (all files) 2024-11-17T22:42:05,459 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/A in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:05,459 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/B in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:05,459 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/3d43113de1b447bba17b4fdeebc1981f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/5b5fa9753947414d8350f5a64d792991, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d91173d93f214deeaba820a0eea547a0] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=34.2 K 2024-11-17T22:42:05,459 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/1bd67d5a96d947c6b92f45d1650e45c8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2f278b9be3ea4db494797ea0af3d20b0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ee38c76653d540cc8087890302246a4b] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=36.5 K 2024-11-17T22:42:05,460 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d43113de1b447bba17b4fdeebc1981f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731883321149 2024-11-17T22:42:05,460 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bd67d5a96d947c6b92f45d1650e45c8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731883321149 2024-11-17T22:42:05,460 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f278b9be3ea4db494797ea0af3d20b0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731883322343 2024-11-17T22:42:05,460 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b5fa9753947414d8350f5a64d792991, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731883322343 2024-11-17T22:42:05,460 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee38c76653d540cc8087890302246a4b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731883323567 2024-11-17T22:42:05,460 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d91173d93f214deeaba820a0eea547a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731883323567 2024-11-17T22:42:05,466 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:05,466 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/91f9df5a38054ee0b5d5f8b5cec5e395 is 50, key is test_row_0/B:col10/1731883324211/Put/seqid=0 2024-11-17T22:42:05,466 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:05,467 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/00bf0de4d3f248fbbb761bdc66bb9a52 is 50, key is test_row_0/A:col10/1731883324211/Put/seqid=0 2024-11-17T22:42:05,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742307_1483 (size=12915) 2024-11-17T22:42:05,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742308_1484 (size=12915) 2024-11-17T22:42:05,480 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/91f9df5a38054ee0b5d5f8b5cec5e395 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/91f9df5a38054ee0b5d5f8b5cec5e395 2024-11-17T22:42:05,485 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/B of 6c99de6ca45c57ddad88306e57b16183 into 91f9df5a38054ee0b5d5f8b5cec5e395(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:05,485 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:05,485 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/B, priority=13, startTime=1731883325458; duration=0sec 2024-11-17T22:42:05,485 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:05,485 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:B 2024-11-17T22:42:05,485 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:05,486 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:05,486 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 6c99de6ca45c57ddad88306e57b16183/C is initiating minor compaction (all files) 2024-11-17T22:42:05,486 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c99de6ca45c57ddad88306e57b16183/C in TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:05,487 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/6b82f50b4f8740f99048f28d4b6db457, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/93cf01355f50415e82666b62896a162a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9a1959f6e1ae46848840561e697b042c] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp, totalSize=34.2 K 2024-11-17T22:42:05,487 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b82f50b4f8740f99048f28d4b6db457, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731883321149 2024-11-17T22:42:05,487 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 93cf01355f50415e82666b62896a162a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731883322343 2024-11-17T22:42:05,487 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a1959f6e1ae46848840561e697b042c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731883323567 2024-11-17T22:42:05,494 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c99de6ca45c57ddad88306e57b16183#C#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:05,495 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/019f9c547f654d42bd0c3c52cf618920 is 50, key is test_row_0/C:col10/1731883324211/Put/seqid=0 2024-11-17T22:42:05,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742309_1485 (size=12915) 2024-11-17T22:42:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T22:42:05,807 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:05,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:42:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-17T22:42:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T22:42:05,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:42:05,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:42:05,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:42:05,879 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/00bf0de4d3f248fbbb761bdc66bb9a52 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/00bf0de4d3f248fbbb761bdc66bb9a52 2024-11-17T22:42:05,883 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/A of 6c99de6ca45c57ddad88306e57b16183 into 00bf0de4d3f248fbbb761bdc66bb9a52(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:05,883 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:05,883 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/A, priority=13, startTime=1731883325458; duration=0sec 2024-11-17T22:42:05,883 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:05,883 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:A 2024-11-17T22:42:05,910 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/019f9c547f654d42bd0c3c52cf618920 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/019f9c547f654d42bd0c3c52cf618920 2024-11-17T22:42:05,915 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c99de6ca45c57ddad88306e57b16183/C of 6c99de6ca45c57ddad88306e57b16183 into 019f9c547f654d42bd0c3c52cf618920(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:05,915 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:05,915 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., storeName=6c99de6ca45c57ddad88306e57b16183/C, priority=13, startTime=1731883325458; duration=0sec 2024-11-17T22:42:05,915 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:05,915 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c99de6ca45c57ddad88306e57b16183:C 2024-11-17T22:42:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T22:42:05,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T22:42:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:05,961 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-17T22:42:05,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/04153221deee450b8baac596f204fa15 is 50, key is test_row_0/A:col10/1731883324232/Put/seqid=0 2024-11-17T22:42:05,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742310_1486 (size=12301) 2024-11-17T22:42:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T22:42:06,359 INFO [Thread-1895 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1895. 2024-11-17T22:42:06,359 DEBUG [Thread-1895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:06,359 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,359 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:06,360 INFO [Thread-1899 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1899. 2024-11-17T22:42:06,360 DEBUG [Thread-1899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:06,360 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,360 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,360 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:06,361 INFO [Thread-1901 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1901. 2024-11-17T22:42:06,361 DEBUG [Thread-1901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:06,361 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,361 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,361 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:06,363 INFO [Thread-1903 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1903. 2024-11-17T22:42:06,363 DEBUG [Thread-1903 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:06,363 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,363 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:06,364 INFO [Thread-1897 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1897. 2024-11-17T22:42:06,364 DEBUG [Thread-1897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:06,364 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,364 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:06,379 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/04153221deee450b8baac596f204fa15 2024-11-17T22:42:06,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/8cc5f357d270472b97432ca9f795daba is 50, key is test_row_0/B:col10/1731883324232/Put/seqid=0 2024-11-17T22:42:06,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742311_1487 (size=12301) 2024-11-17T22:42:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:42:06,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. as already flushing 2024-11-17T22:42:06,400 INFO [Thread-1888 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1888. 2024-11-17T22:42:06,400 DEBUG [Thread-1888 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:06,400 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,400 INFO [Thread-1890 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1890. 2024-11-17T22:42:06,400 DEBUG [Thread-1890 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:06,400 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,400 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:06,400 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:06,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T22:42:06,790 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/8cc5f357d270472b97432ca9f795daba 2024-11-17T22:42:06,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/dc26fa5a10474ecc900d6f1e02742ca3 is 50, key is test_row_0/C:col10/1731883324232/Put/seqid=0 2024-11-17T22:42:06,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742312_1488 (size=12301) 2024-11-17T22:42:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T22:42:07,208 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/dc26fa5a10474ecc900d6f1e02742ca3 2024-11-17T22:42:07,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/04153221deee450b8baac596f204fa15 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/04153221deee450b8baac596f204fa15 2024-11-17T22:42:07,216 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/04153221deee450b8baac596f204fa15, entries=150, sequenceid=322, filesize=12.0 K 2024-11-17T22:42:07,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/8cc5f357d270472b97432ca9f795daba as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/8cc5f357d270472b97432ca9f795daba 2024-11-17T22:42:07,220 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/8cc5f357d270472b97432ca9f795daba, entries=150, sequenceid=322, filesize=12.0 K 2024-11-17T22:42:07,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/dc26fa5a10474ecc900d6f1e02742ca3 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/dc26fa5a10474ecc900d6f1e02742ca3 2024-11-17T22:42:07,225 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/dc26fa5a10474ecc900d6f1e02742ca3, entries=150, sequenceid=322, filesize=12.0 K 2024-11-17T22:42:07,226 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=13.42 KB/13740 for 6c99de6ca45c57ddad88306e57b16183 in 1265ms, sequenceid=322, compaction requested=false 2024-11-17T22:42:07,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 6c99de6ca45c57ddad88306e57b16183: 2024-11-17T22:42:07,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:07,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-17T22:42:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-17T22:42:07,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-17T22:42:07,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4190 sec 2024-11-17T22:42:07,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.4230 sec 2024-11-17T22:42:07,532 INFO [Thread-1892 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1892. 2024-11-17T22:42:07,533 DEBUG [Thread-1892 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:07,533 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:07,533 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:07,533 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:07,569 INFO [Thread-1886 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1886. 2024-11-17T22:42:07,569 DEBUG [Thread-1886 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:07,569 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:07,570 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:07,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:07,579 INFO [Thread-1884 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1884. 2024-11-17T22:42:07,579 DEBUG [Thread-1884 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:07,579 DEBUG [Thread-1884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:07,579 DEBUG [Thread-1884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:07,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:07,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T22:42:07,958 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 150 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1518 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4554 rows 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1523 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4569 rows 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1520 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4557 rows 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1522 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4566 rows 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1523 2024-11-17T22:42:07,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4568 rows 2024-11-17T22:42:07,958 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T22:42:07,959 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T22:42:07,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:07,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:07,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:07,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T22:42:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T22:42:07,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883327965"}]},"ts":"1731883327965"} 2024-11-17T22:42:07,968 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T22:42:07,968 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T22:42:07,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T22:42:07,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c99de6ca45c57ddad88306e57b16183, UNASSIGN}] 2024-11-17T22:42:07,972 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c99de6ca45c57ddad88306e57b16183, UNASSIGN 2024-11-17T22:42:07,973 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=6c99de6ca45c57ddad88306e57b16183, regionState=CLOSING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:07,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c99de6ca45c57ddad88306e57b16183, UNASSIGN because future has completed 2024-11-17T22:42:07,975 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T22:42:07,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:42:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T22:42:08,129 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:42:08,129 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T22:42:08,129 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 6c99de6ca45c57ddad88306e57b16183, disabling compactions & flushes 2024-11-17T22:42:08,129 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:08,129 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:08,129 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. after waiting 0 ms 2024-11-17T22:42:08,129 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:08,130 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 6c99de6ca45c57ddad88306e57b16183 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T22:42:08,138 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/7218c4bf0d664e82bd77fd44e4d6a6d2 is 50, key is test_row_0/A:col10/1731883327568/Put/seqid=0 2024-11-17T22:42:08,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742313_1489 (size=9857) 2024-11-17T22:42:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T22:42:08,542 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/7218c4bf0d664e82bd77fd44e4d6a6d2 2024-11-17T22:42:08,550 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/81b399c7d9a942e1974ca7ae3b2ab9e5 is 50, key is test_row_0/B:col10/1731883327568/Put/seqid=0 2024-11-17T22:42:08,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742314_1490 (size=9857) 2024-11-17T22:42:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T22:42:08,955 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/81b399c7d9a942e1974ca7ae3b2ab9e5 2024-11-17T22:42:08,964 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/8793f85d29c84ef2af2a6adfdcad4023 is 50, key is test_row_0/C:col10/1731883327568/Put/seqid=0 2024-11-17T22:42:08,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742315_1491 (size=9857) 2024-11-17T22:42:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T22:42:09,370 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/8793f85d29c84ef2af2a6adfdcad4023 2024-11-17T22:42:09,381 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/A/7218c4bf0d664e82bd77fd44e4d6a6d2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/7218c4bf0d664e82bd77fd44e4d6a6d2 2024-11-17T22:42:09,385 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/7218c4bf0d664e82bd77fd44e4d6a6d2, entries=100, sequenceid=330, filesize=9.6 K 2024-11-17T22:42:09,386 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/B/81b399c7d9a942e1974ca7ae3b2ab9e5 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/81b399c7d9a942e1974ca7ae3b2ab9e5 2024-11-17T22:42:09,390 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/81b399c7d9a942e1974ca7ae3b2ab9e5, entries=100, sequenceid=330, filesize=9.6 K 2024-11-17T22:42:09,391 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/.tmp/C/8793f85d29c84ef2af2a6adfdcad4023 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/8793f85d29c84ef2af2a6adfdcad4023 2024-11-17T22:42:09,395 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/8793f85d29c84ef2af2a6adfdcad4023, entries=100, sequenceid=330, filesize=9.6 K 2024-11-17T22:42:09,396 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 6c99de6ca45c57ddad88306e57b16183 in 1267ms, sequenceid=330, compaction requested=true 2024-11-17T22:42:09,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/614cdc28137a43c2add90f3cca6a94a9, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83dbed8c6ecb4f72aafddb33c57df731, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2e81a38ddc8b4e9da6de6d91c8aa90ef, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/dcdd8e44d6e7475e96af38b870057e81, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/d8c1c8c402d34acf91ec8e1f8f57a994, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/eed4c993f69747c5a428a33cb6a2f1a2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83418ae4b105464696b0e5e177d64b24, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/126452f9b58143fb944d5b5afa866a5d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/8b884cf1d6f0408abcc2c19a6ccf9f0d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/c797f91f60284a02ab72740499b95468, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/9c13adb52e00489090c1ddecde733745, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f57aaa2b5fee49ceb7f512cb2930dbcc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f8386ee4714a40d7b652965917e2ce33, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ae90b111b90441d3bd1376ed5ef99e88, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/24e3deec1f9d470999bcc00c29565ba1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/1bd67d5a96d947c6b92f45d1650e45c8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2f278b9be3ea4db494797ea0af3d20b0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ee38c76653d540cc8087890302246a4b] to archive 2024-11-17T22:42:09,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:42:09,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/614cdc28137a43c2add90f3cca6a94a9 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/614cdc28137a43c2add90f3cca6a94a9 2024-11-17T22:42:09,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83dbed8c6ecb4f72aafddb33c57df731 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83dbed8c6ecb4f72aafddb33c57df731 2024-11-17T22:42:09,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2e81a38ddc8b4e9da6de6d91c8aa90ef to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2e81a38ddc8b4e9da6de6d91c8aa90ef 2024-11-17T22:42:09,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/dcdd8e44d6e7475e96af38b870057e81 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/dcdd8e44d6e7475e96af38b870057e81 2024-11-17T22:42:09,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/d8c1c8c402d34acf91ec8e1f8f57a994 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/d8c1c8c402d34acf91ec8e1f8f57a994 2024-11-17T22:42:09,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/eed4c993f69747c5a428a33cb6a2f1a2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/eed4c993f69747c5a428a33cb6a2f1a2 2024-11-17T22:42:09,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83418ae4b105464696b0e5e177d64b24 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/83418ae4b105464696b0e5e177d64b24 2024-11-17T22:42:09,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/126452f9b58143fb944d5b5afa866a5d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/126452f9b58143fb944d5b5afa866a5d 2024-11-17T22:42:09,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/8b884cf1d6f0408abcc2c19a6ccf9f0d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/8b884cf1d6f0408abcc2c19a6ccf9f0d 2024-11-17T22:42:09,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/c797f91f60284a02ab72740499b95468 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/c797f91f60284a02ab72740499b95468 2024-11-17T22:42:09,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/9c13adb52e00489090c1ddecde733745 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/9c13adb52e00489090c1ddecde733745 2024-11-17T22:42:09,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f57aaa2b5fee49ceb7f512cb2930dbcc to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f57aaa2b5fee49ceb7f512cb2930dbcc 2024-11-17T22:42:09,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f8386ee4714a40d7b652965917e2ce33 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/f8386ee4714a40d7b652965917e2ce33 2024-11-17T22:42:09,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ae90b111b90441d3bd1376ed5ef99e88 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ae90b111b90441d3bd1376ed5ef99e88 2024-11-17T22:42:09,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/24e3deec1f9d470999bcc00c29565ba1 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/24e3deec1f9d470999bcc00c29565ba1 2024-11-17T22:42:09,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/1bd67d5a96d947c6b92f45d1650e45c8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/1bd67d5a96d947c6b92f45d1650e45c8 2024-11-17T22:42:09,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2f278b9be3ea4db494797ea0af3d20b0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/2f278b9be3ea4db494797ea0af3d20b0 2024-11-17T22:42:09,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ee38c76653d540cc8087890302246a4b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/ee38c76653d540cc8087890302246a4b 2024-11-17T22:42:09,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/031fae5be3e649bf8bcea7d6c356c081, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/537382436e5843ce8f5231b1019a156a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/b81b890a3c574362b66a52c6abe1f03f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/cf30ffa8b88142a39104d75a41368ced, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/db631484d9564c5ab8aea6f04e3f4a60, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d36bb40d45724797adb188609d538fb0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/28293dfc84b644e18f70e2c2893cc809, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/67db2355f0f34d73a1ac9065b8fdc278, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/b5b25b9e2fed47c189e7bb9a54050e12, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/09fc08c533dd4695987ce76102ab9996, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/95013405c31c4667b7a1c994ea49d257, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/557e520893dc406689600e6d6c28983a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/9df8485bf72b43a5b4ac35ceec3ad6d6, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/fbc32884c6264fca9e234b420d26253b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/3d43113de1b447bba17b4fdeebc1981f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/e0995f062bb643e7933ead9a83ea1ef0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/5b5fa9753947414d8350f5a64d792991, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d91173d93f214deeaba820a0eea547a0] to archive 2024-11-17T22:42:09,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:42:09,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/031fae5be3e649bf8bcea7d6c356c081 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/031fae5be3e649bf8bcea7d6c356c081 2024-11-17T22:42:09,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/537382436e5843ce8f5231b1019a156a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/537382436e5843ce8f5231b1019a156a 2024-11-17T22:42:09,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/b81b890a3c574362b66a52c6abe1f03f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/b81b890a3c574362b66a52c6abe1f03f 2024-11-17T22:42:09,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/cf30ffa8b88142a39104d75a41368ced to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/cf30ffa8b88142a39104d75a41368ced 2024-11-17T22:42:09,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/db631484d9564c5ab8aea6f04e3f4a60 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/db631484d9564c5ab8aea6f04e3f4a60 2024-11-17T22:42:09,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d36bb40d45724797adb188609d538fb0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d36bb40d45724797adb188609d538fb0 2024-11-17T22:42:09,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/28293dfc84b644e18f70e2c2893cc809 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/28293dfc84b644e18f70e2c2893cc809 2024-11-17T22:42:09,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/67db2355f0f34d73a1ac9065b8fdc278 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/67db2355f0f34d73a1ac9065b8fdc278 2024-11-17T22:42:09,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/b5b25b9e2fed47c189e7bb9a54050e12 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/b5b25b9e2fed47c189e7bb9a54050e12 2024-11-17T22:42:09,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/09fc08c533dd4695987ce76102ab9996 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/09fc08c533dd4695987ce76102ab9996 2024-11-17T22:42:09,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/95013405c31c4667b7a1c994ea49d257 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/95013405c31c4667b7a1c994ea49d257 2024-11-17T22:42:09,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/557e520893dc406689600e6d6c28983a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/557e520893dc406689600e6d6c28983a 2024-11-17T22:42:09,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/9df8485bf72b43a5b4ac35ceec3ad6d6 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/9df8485bf72b43a5b4ac35ceec3ad6d6 2024-11-17T22:42:09,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/fbc32884c6264fca9e234b420d26253b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/fbc32884c6264fca9e234b420d26253b 2024-11-17T22:42:09,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/3d43113de1b447bba17b4fdeebc1981f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/3d43113de1b447bba17b4fdeebc1981f 2024-11-17T22:42:09,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/e0995f062bb643e7933ead9a83ea1ef0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/e0995f062bb643e7933ead9a83ea1ef0 2024-11-17T22:42:09,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/5b5fa9753947414d8350f5a64d792991 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/5b5fa9753947414d8350f5a64d792991 2024-11-17T22:42:09,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d91173d93f214deeaba820a0eea547a0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/d91173d93f214deeaba820a0eea547a0 2024-11-17T22:42:09,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/04efb083e0d341d89001a87b747de180, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/14ef6377ed034f47bc7650bd42558a34, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d69600fb740248f99665c149abc0b00b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9b370b0099da47c6a5ffd5504d3fe8d8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/55769458fd15436b80776abf1ff72bab, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/16ed9a4632ee4142b8570ebbf16ac3cc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c37c116d141f4b6b92578b5a01a5a8dd, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/691968656b9a458daf327d4b9a859b3e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/230b73fb90a946c3b54ee39d7a5f511f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9fa193d0674d46b28f0870c7c572481f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/e0aae51e743241508dd540eda3d4b474, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/5c4b269e20ea429eb9e72855c8abb279, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/92467af6b9134c81a7582a451affa476, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/22059bc6b58a4b1c9b0d9e6ce56734a0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/6b82f50b4f8740f99048f28d4b6db457, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d0a7a6d426f2490fb88c9c619c23b714, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/93cf01355f50415e82666b62896a162a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9a1959f6e1ae46848840561e697b042c] to archive 2024-11-17T22:42:09,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:42:09,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/04efb083e0d341d89001a87b747de180 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/04efb083e0d341d89001a87b747de180 2024-11-17T22:42:09,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/14ef6377ed034f47bc7650bd42558a34 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/14ef6377ed034f47bc7650bd42558a34 2024-11-17T22:42:09,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d69600fb740248f99665c149abc0b00b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d69600fb740248f99665c149abc0b00b 2024-11-17T22:42:09,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9b370b0099da47c6a5ffd5504d3fe8d8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9b370b0099da47c6a5ffd5504d3fe8d8 2024-11-17T22:42:09,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/55769458fd15436b80776abf1ff72bab to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/55769458fd15436b80776abf1ff72bab 2024-11-17T22:42:09,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/16ed9a4632ee4142b8570ebbf16ac3cc to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/16ed9a4632ee4142b8570ebbf16ac3cc 2024-11-17T22:42:09,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c37c116d141f4b6b92578b5a01a5a8dd to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/c37c116d141f4b6b92578b5a01a5a8dd 2024-11-17T22:42:09,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/691968656b9a458daf327d4b9a859b3e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/691968656b9a458daf327d4b9a859b3e 2024-11-17T22:42:09,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/230b73fb90a946c3b54ee39d7a5f511f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/230b73fb90a946c3b54ee39d7a5f511f 2024-11-17T22:42:09,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9fa193d0674d46b28f0870c7c572481f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9fa193d0674d46b28f0870c7c572481f 2024-11-17T22:42:09,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/e0aae51e743241508dd540eda3d4b474 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/e0aae51e743241508dd540eda3d4b474 2024-11-17T22:42:09,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/5c4b269e20ea429eb9e72855c8abb279 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/5c4b269e20ea429eb9e72855c8abb279 2024-11-17T22:42:09,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/92467af6b9134c81a7582a451affa476 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/92467af6b9134c81a7582a451affa476 2024-11-17T22:42:09,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/22059bc6b58a4b1c9b0d9e6ce56734a0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/22059bc6b58a4b1c9b0d9e6ce56734a0 2024-11-17T22:42:09,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/6b82f50b4f8740f99048f28d4b6db457 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/6b82f50b4f8740f99048f28d4b6db457 2024-11-17T22:42:09,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d0a7a6d426f2490fb88c9c619c23b714 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/d0a7a6d426f2490fb88c9c619c23b714 2024-11-17T22:42:09,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/93cf01355f50415e82666b62896a162a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/93cf01355f50415e82666b62896a162a 2024-11-17T22:42:09,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9a1959f6e1ae46848840561e697b042c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/9a1959f6e1ae46848840561e697b042c 2024-11-17T22:42:09,494 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/recovered.edits/333.seqid, newMaxSeqId=333, maxSeqId=1 2024-11-17T22:42:09,495 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. 2024-11-17T22:42:09,495 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 6c99de6ca45c57ddad88306e57b16183: Waiting for close lock at 1731883328129Running coprocessor pre-close hooks at 1731883328129Disabling compacts and flushes for region at 1731883328129Disabling writes for close at 1731883328129Obtaining lock to block concurrent updates at 1731883328130 (+1 ms)Preparing flush snapshotting stores in 6c99de6ca45c57ddad88306e57b16183 at 1731883328130Finished memstore snapshotting TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731883328131 (+1 ms)Flushing stores of TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183. at 1731883328132 (+1 ms)Flushing 6c99de6ca45c57ddad88306e57b16183/A: creating writer at 1731883328132Flushing 6c99de6ca45c57ddad88306e57b16183/A: appending metadata at 1731883328138 (+6 ms)Flushing 6c99de6ca45c57ddad88306e57b16183/A: closing flushed file at 1731883328138Flushing 6c99de6ca45c57ddad88306e57b16183/B: creating writer at 1731883328547 (+409 ms)Flushing 6c99de6ca45c57ddad88306e57b16183/B: appending metadata at 1731883328549 (+2 ms)Flushing 6c99de6ca45c57ddad88306e57b16183/B: closing flushed file at 1731883328549Flushing 6c99de6ca45c57ddad88306e57b16183/C: creating writer at 1731883328960 (+411 ms)Flushing 6c99de6ca45c57ddad88306e57b16183/C: appending metadata at 1731883328964 (+4 ms)Flushing 6c99de6ca45c57ddad88306e57b16183/C: closing flushed file at 1731883328964Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e7c68cd: reopening flushed file at 1731883329379 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c6822ab: reopening flushed file at 1731883329385 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53b3cc3e: reopening flushed file at 1731883329390 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 6c99de6ca45c57ddad88306e57b16183 in 1267ms, sequenceid=330, compaction requested=true at 1731883329396 (+6 ms)Writing region close event to WAL at 1731883329490 (+94 ms)Running coprocessor post-close hooks at 1731883329495 (+5 ms)Closed at 1731883329495 2024-11-17T22:42:09,498 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:42:09,499 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=6c99de6ca45c57ddad88306e57b16183, regionState=CLOSED 2024-11-17T22:42:09,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:42:09,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-17T22:42:09,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 6c99de6ca45c57ddad88306e57b16183, server=1a6e40b21a48,43011,1731883188895 in 1.5270 sec 2024-11-17T22:42:09,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-17T22:42:09,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c99de6ca45c57ddad88306e57b16183, UNASSIGN in 1.5340 sec 2024-11-17T22:42:09,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-17T22:42:09,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5390 sec 2024-11-17T22:42:09,513 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883329512"}]},"ts":"1731883329512"} 2024-11-17T22:42:09,515 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T22:42:09,515 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T22:42:09,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5560 sec 2024-11-17T22:42:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T22:42:10,107 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:10,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T22:42:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:10,109 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T22:42:10,110 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:10,112 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:42:10,114 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/recovered.edits] 2024-11-17T22:42:10,116 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/00bf0de4d3f248fbbb761bdc66bb9a52 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/00bf0de4d3f248fbbb761bdc66bb9a52 2024-11-17T22:42:10,117 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/04153221deee450b8baac596f204fa15 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/04153221deee450b8baac596f204fa15 2024-11-17T22:42:10,118 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/7218c4bf0d664e82bd77fd44e4d6a6d2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/A/7218c4bf0d664e82bd77fd44e4d6a6d2 2024-11-17T22:42:10,119 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/81b399c7d9a942e1974ca7ae3b2ab9e5 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/81b399c7d9a942e1974ca7ae3b2ab9e5 2024-11-17T22:42:10,120 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/8cc5f357d270472b97432ca9f795daba to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/8cc5f357d270472b97432ca9f795daba 2024-11-17T22:42:10,121 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/91f9df5a38054ee0b5d5f8b5cec5e395 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/B/91f9df5a38054ee0b5d5f8b5cec5e395 2024-11-17T22:42:10,122 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/019f9c547f654d42bd0c3c52cf618920 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/019f9c547f654d42bd0c3c52cf618920 2024-11-17T22:42:10,123 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/8793f85d29c84ef2af2a6adfdcad4023 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/8793f85d29c84ef2af2a6adfdcad4023 2024-11-17T22:42:10,123 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/dc26fa5a10474ecc900d6f1e02742ca3 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/C/dc26fa5a10474ecc900d6f1e02742ca3 2024-11-17T22:42:10,125 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/recovered.edits/333.seqid to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183/recovered.edits/333.seqid 2024-11-17T22:42:10,126 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/6c99de6ca45c57ddad88306e57b16183 2024-11-17T22:42:10,126 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T22:42:10,127 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:10,129 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T22:42:10,131 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T22:42:10,132 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:10,132 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T22:42:10,132 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731883330132"}]},"ts":"9223372036854775807"} 2024-11-17T22:42:10,133 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T22:42:10,133 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6c99de6ca45c57ddad88306e57b16183, NAME => 'TestAcidGuarantees,,1731883304144.6c99de6ca45c57ddad88306e57b16183.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T22:42:10,133 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T22:42:10,133 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731883330133"}]},"ts":"9223372036854775807"} 2024-11-17T22:42:10,135 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T22:42:10,136 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:10,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-17T22:42:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T22:42:10,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T22:42:10,222 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:10,233 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 240), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=356 (was 403), ProcessCount=11 (was 11), AvailableMemoryMB=4297 (was 4445) 2024-11-17T22:42:10,243 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=356, ProcessCount=11, AvailableMemoryMB=4297 2024-11-17T22:42:10,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T22:42:10,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T22:42:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:10,247 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T22:42:10,247 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:10,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-17T22:42:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T22:42:10,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T22:42:10,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742316_1492 (size=959) 2024-11-17T22:42:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T22:42:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T22:42:10,655 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71 2024-11-17T22:42:10,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742317_1493 (size=53) 2024-11-17T22:42:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T22:42:11,061 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:42:11,061 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1e5636f3ed44c24ae3f10ff764ecff86, disabling compactions & flushes 2024-11-17T22:42:11,061 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:11,061 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:11,061 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. after waiting 0 ms 2024-11-17T22:42:11,061 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:11,061 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:11,061 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e5636f3ed44c24ae3f10ff764ecff86: Waiting for close lock at 1731883331061Disabling compacts and flushes for region at 1731883331061Disabling writes for close at 1731883331061Writing region close event to WAL at 1731883331061Closed at 1731883331061 2024-11-17T22:42:11,062 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T22:42:11,062 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731883331062"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731883331062"}]},"ts":"1731883331062"} 2024-11-17T22:42:11,064 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T22:42:11,065 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T22:42:11,066 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883331065"}]},"ts":"1731883331065"} 2024-11-17T22:42:11,068 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T22:42:11,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, ASSIGN}] 2024-11-17T22:42:11,070 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, ASSIGN 2024-11-17T22:42:11,071 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, ASSIGN; state=OFFLINE, location=1a6e40b21a48,43011,1731883188895; forceNewPlan=false, retain=false 2024-11-17T22:42:11,222 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=1e5636f3ed44c24ae3f10ff764ecff86, regionState=OPENING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:11,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, ASSIGN because future has completed 2024-11-17T22:42:11,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:42:11,385 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:11,386 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} 2024-11-17T22:42:11,386 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,386 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:42:11,386 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,386 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T22:42:11,388 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,390 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e5636f3ed44c24ae3f10ff764ecff86 columnFamilyName A 2024-11-17T22:42:11,390 DEBUG [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:11,390 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(327): Store=1e5636f3ed44c24ae3f10ff764ecff86/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:42:11,391 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,392 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e5636f3ed44c24ae3f10ff764ecff86 columnFamilyName B 2024-11-17T22:42:11,392 DEBUG [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:11,393 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(327): Store=1e5636f3ed44c24ae3f10ff764ecff86/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:42:11,393 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,395 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e5636f3ed44c24ae3f10ff764ecff86 columnFamilyName C 2024-11-17T22:42:11,395 DEBUG [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:11,396 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(327): Store=1e5636f3ed44c24ae3f10ff764ecff86/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:42:11,396 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,397 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,397 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,399 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,399 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,400 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T22:42:11,402 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,405 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T22:42:11,406 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 1e5636f3ed44c24ae3f10ff764ecff86; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61585529, jitterRate=-0.08230410516262054}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T22:42:11,406 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:11,407 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 1e5636f3ed44c24ae3f10ff764ecff86: Running coprocessor pre-open hook at 1731883331386Writing region info on filesystem at 1731883331386Initializing all the Stores at 1731883331387 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883331387Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883331388 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883331388Cleaning up temporary data from old regions at 1731883331399 (+11 ms)Running coprocessor post-open hooks at 1731883331406 (+7 ms)Region opened successfully at 1731883331407 (+1 ms) 2024-11-17T22:42:11,408 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., pid=148, masterSystemTime=1731883331382 2024-11-17T22:42:11,410 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:11,410 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:11,410 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=1e5636f3ed44c24ae3f10ff764ecff86, regionState=OPEN, openSeqNum=2, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:11,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:42:11,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-17T22:42:11,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 in 185 msec 2024-11-17T22:42:11,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-17T22:42:11,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, ASSIGN in 346 msec 2024-11-17T22:42:11,417 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T22:42:11,417 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883331417"}]},"ts":"1731883331417"} 2024-11-17T22:42:11,419 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T22:42:11,419 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T22:42:11,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-17T22:42:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T22:42:12,408 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:12,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43602cdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:12,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:42:12,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:42:12,412 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:42:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:42:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:42:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65bbf255, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:42:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:42:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:12,414 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:42:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10aba9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:12,416 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:42:12,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:42:12,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:42:12,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:42:12,419 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T22:42:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a6e40b21a48,45671,1731883188234 2024-11-17T22:42:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@29cfa3dd 2024-11-17T22:42:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T22:42:12,421 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T22:42:12,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T22:42:12,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T22:42:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:12,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742318_1494 (size=995) 2024-11-17T22:42:12,837 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T22:42:12,837 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T22:42:12,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T22:42:12,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, REOPEN/MOVE}] 2024-11-17T22:42:12,845 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, REOPEN/MOVE 2024-11-17T22:42:12,846 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=1e5636f3ed44c24ae3f10ff764ecff86, regionState=CLOSING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:12,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, REOPEN/MOVE because future has completed 2024-11-17T22:42:12,848 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T22:42:12,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:42:13,001 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,001 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T22:42:13,001 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 1e5636f3ed44c24ae3f10ff764ecff86, disabling compactions & flushes 2024-11-17T22:42:13,001 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,001 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,001 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. after waiting 0 ms 2024-11-17T22:42:13,001 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,004 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T22:42:13,005 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,005 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 1e5636f3ed44c24ae3f10ff764ecff86: Waiting for close lock at 1731883333001Running coprocessor pre-close hooks at 1731883333001Disabling compacts and flushes for region at 1731883333001Disabling writes for close at 1731883333001Writing region close event to WAL at 1731883333001Running coprocessor post-close hooks at 1731883333004 (+3 ms)Closed at 1731883333005 (+1 ms) 2024-11-17T22:42:13,005 WARN [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 1e5636f3ed44c24ae3f10ff764ecff86 to self. 2024-11-17T22:42:13,006 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,007 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=1e5636f3ed44c24ae3f10ff764ecff86, regionState=CLOSED 2024-11-17T22:42:13,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:42:13,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-17T22:42:13,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 in 160 msec 2024-11-17T22:42:13,010 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, REOPEN/MOVE; state=CLOSED, location=1a6e40b21a48,43011,1731883188895; forceNewPlan=false, retain=true 2024-11-17T22:42:13,161 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=1e5636f3ed44c24ae3f10ff764ecff86, regionState=OPENING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, REOPEN/MOVE because future has completed 2024-11-17T22:42:13,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:42:13,324 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,325 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} 2024-11-17T22:42:13,325 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,325 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T22:42:13,325 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,325 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,328 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,329 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e5636f3ed44c24ae3f10ff764ecff86 columnFamilyName A 2024-11-17T22:42:13,330 DEBUG [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:13,331 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(327): Store=1e5636f3ed44c24ae3f10ff764ecff86/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:42:13,332 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,333 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e5636f3ed44c24ae3f10ff764ecff86 columnFamilyName B 2024-11-17T22:42:13,333 DEBUG [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:13,333 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(327): Store=1e5636f3ed44c24ae3f10ff764ecff86/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:42:13,333 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,334 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e5636f3ed44c24ae3f10ff764ecff86 columnFamilyName C 2024-11-17T22:42:13,334 DEBUG [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:13,335 INFO [StoreOpener-1e5636f3ed44c24ae3f10ff764ecff86-1 {}] regionserver.HStore(327): Store=1e5636f3ed44c24ae3f10ff764ecff86/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T22:42:13,335 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,336 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,338 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,339 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,339 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,340 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T22:42:13,342 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,343 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 1e5636f3ed44c24ae3f10ff764ecff86; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63190693, jitterRate=-0.058385297656059265}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T22:42:13,343 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,343 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 1e5636f3ed44c24ae3f10ff764ecff86: Running coprocessor pre-open hook at 1731883333326Writing region info on filesystem at 1731883333326Initializing all the Stores at 1731883333327 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883333327Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883333327Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731883333327Cleaning up temporary data from old regions at 1731883333339 (+12 ms)Running coprocessor post-open hooks at 1731883333343 (+4 ms)Region opened successfully at 1731883333343 2024-11-17T22:42:13,344 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., pid=153, masterSystemTime=1731883333317 2024-11-17T22:42:13,345 DEBUG [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,345 INFO [RS_OPEN_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,346 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=1e5636f3ed44c24ae3f10ff764ecff86, regionState=OPEN, openSeqNum=5, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:42:13,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-17T22:42:13,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 in 184 msec 2024-11-17T22:42:13,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-17T22:42:13,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, REOPEN/MOVE in 506 msec 2024-11-17T22:42:13,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-17T22:42:13,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 513 msec 2024-11-17T22:42:13,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 931 msec 2024-11-17T22:42:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-17T22:42:13,357 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:13,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e76e6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:42:13,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:42:13,359 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:42:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:42:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:42:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b0de20c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:42:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:42:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:13,361 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:42:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d4adbbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:42:13,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:42:13,364 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:42:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:42:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:42:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@353a7f74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:42:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:42:13,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:13,365 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:42:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dac3bb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,367 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:42:13,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:42:13,368 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:42:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:42:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:42:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61293f30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:42:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:42:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:13,370 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:42:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@678af18f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,371 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:42:13,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:42:13,372 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:42:13,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:42:13,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:42:13,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb99a90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:42:13,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:42:13,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:13,374 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:42:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@231d619f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:42:13,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:42:13,377 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:42:13,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:42:13,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:42:13,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f05d1b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:42:13,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:42:13,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:13,379 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:42:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b932cc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:42:13,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:42:13,381 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:42:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:42:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:42:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379325c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:42:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:42:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:13,382 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:42:13,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8deac6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:42:13,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:42:13,384 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:42:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:42:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:42:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a79592, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:42:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:42:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:13,385 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:42:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1233a84e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:42:13,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:42:13,387 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:42:13,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:42:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:42:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa700c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:42:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:42:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:13,389 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:42:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62a652e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:42:13,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:42:13,391 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:42:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:42:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:42:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f5997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:42:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:42:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:13,392 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:42:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5237fc05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a6e40b21a48,45671,-1 for getting cluster id 2024-11-17T22:42:13,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T22:42:13,394 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a900e0c-d2b7-4958-a2a4-2ef9619222c7' 2024-11-17T22:42:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T22:42:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a900e0c-d2b7-4958-a2a4-2ef9619222c7" 2024-11-17T22:42:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3f4f18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a6e40b21a48,45671,-1] 2024-11-17T22:42:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T22:42:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:13,395 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T22:42:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be937ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T22:42:13,396 DEBUG [Thread-2212 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:42:13,397 DEBUG [Thread-2223 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:42:13,397 DEBUG [Thread-2225 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:42:13,397 DEBUG [Thread-2216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:42:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:42:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:42:13,397 DEBUG [Thread-2229 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:42:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:42:13,397 DEBUG [Thread-2218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:42:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:42:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:42:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:42:13,398 DEBUG [Thread-2227 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:42:13,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:42:13,398 DEBUG [Thread-2214 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:42:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:42:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:42:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:42:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:42:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:42:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:42:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:42:13,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:42:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:42:13,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:42:13,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:42:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:42:13,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:42:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:42:13,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:42:13,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:42:13,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:42:13,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:42:13,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:42:13,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:42:13,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:42:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:42:13,403 DEBUG [Thread-2220 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:42:13,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:42:13,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:42:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:42:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:42:13,405 DEBUG [Thread-2231 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T22:42:13,406 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:42:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a6e40b21a48,43011,1731883188895, seqNum=-1] 2024-11-17T22:42:13,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:42:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T22:42:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:42:13,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T22:42:13,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:42:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-17T22:42:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T22:42:13,409 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:42:13,410 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:42:13,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:42:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,413 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:13,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5] 2024-11-17T22:42:13,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:42:13,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117632d417b74894480be614efb6ed1b619_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883333409/Put/seqid=0 2024-11-17T22:42:13,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742319_1495 (size=12154) 2024-11-17T22:42:13,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883393474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883393476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883393477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883393480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883393480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T22:42:13,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T22:42:13,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:13,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:13,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:13,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883393587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883393587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883393588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883393588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883393597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T22:42:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T22:42:13,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883393797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883393797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883393797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883393797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883393806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:13,845 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:13,849 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117632d417b74894480be614efb6ed1b619_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117632d417b74894480be614efb6ed1b619_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:13,850 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/7b60c22263554005b6650cb1c5ed756e, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:13,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/7b60c22263554005b6650cb1c5ed756e is 175, key is test_row_0/A:col10/1731883333409/Put/seqid=0 2024-11-17T22:42:13,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T22:42:13,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:13,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:13,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:13,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:13,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742320_1496 (size=30955) 2024-11-17T22:42:13,876 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/7b60c22263554005b6650cb1c5ed756e 2024-11-17T22:42:13,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/cd7476ef8998443c834d71e112b2b8c2 is 50, key is test_row_0/B:col10/1731883333409/Put/seqid=0 2024-11-17T22:42:13,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742321_1497 (size=12001) 2024-11-17T22:42:13,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/cd7476ef8998443c834d71e112b2b8c2 2024-11-17T22:42:13,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/911d5ce95ff1401dabec9a58c76af4a4 is 50, key is test_row_0/C:col10/1731883333409/Put/seqid=0 2024-11-17T22:42:13,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742322_1498 (size=12001) 2024-11-17T22:42:14,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T22:42:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:14,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:14,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T22:42:14,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883394107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883394107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883394107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883394107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883394116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T22:42:14,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:14,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:14,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T22:42:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:14,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:14,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/911d5ce95ff1401dabec9a58c76af4a4 2024-11-17T22:42:14,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/7b60c22263554005b6650cb1c5ed756e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/7b60c22263554005b6650cb1c5ed756e 2024-11-17T22:42:14,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/7b60c22263554005b6650cb1c5ed756e, entries=150, sequenceid=16, filesize=30.2 K 2024-11-17T22:42:14,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/cd7476ef8998443c834d71e112b2b8c2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cd7476ef8998443c834d71e112b2b8c2 2024-11-17T22:42:14,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cd7476ef8998443c834d71e112b2b8c2, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T22:42:14,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/911d5ce95ff1401dabec9a58c76af4a4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/911d5ce95ff1401dabec9a58c76af4a4 2024-11-17T22:42:14,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/911d5ce95ff1401dabec9a58c76af4a4, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T22:42:14,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1e5636f3ed44c24ae3f10ff764ecff86 in 985ms, sequenceid=16, compaction requested=false 2024-11-17T22:42:14,398 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T22:42:14,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:14,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T22:42:14,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:14,481 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:42:14,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e3568e0596a5436182586d2af525022e_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883333478/Put/seqid=0 2024-11-17T22:42:14,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742323_1499 (size=12154) 2024-11-17T22:42:14,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:14,513 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e3568e0596a5436182586d2af525022e_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e3568e0596a5436182586d2af525022e_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:14,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/bd761915fef04406914b93eac024b966, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:14,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/bd761915fef04406914b93eac024b966 is 175, key is test_row_0/A:col10/1731883333478/Put/seqid=0 2024-11-17T22:42:14,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742324_1500 (size=30955) 2024-11-17T22:42:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T22:42:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:14,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:14,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883394630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883394635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883394642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883394643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883394643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883394747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883394747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883394756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883394757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883394757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,927 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/bd761915fef04406914b93eac024b966 2024-11-17T22:42:14,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/faff33ae15054d819e657bea6fa65a27 is 50, key is test_row_0/B:col10/1731883333478/Put/seqid=0 2024-11-17T22:42:14,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883394957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883394957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883394967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883394967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883394967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:14,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742325_1501 (size=12001) 2024-11-17T22:42:15,022 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T22:42:15,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883395267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:15,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883395267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:15,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883395277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:15,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:15,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883395278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883395277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:15,376 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/faff33ae15054d819e657bea6fa65a27 2024-11-17T22:42:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/218f68fb9063448cb864b2124db2567f is 50, key is test_row_0/C:col10/1731883333478/Put/seqid=0 2024-11-17T22:42:15,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742326_1502 (size=12001) 2024-11-17T22:42:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T22:42:15,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883395776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883395777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:15,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883395787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:15,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:15,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883395798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883395797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:15,855 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/218f68fb9063448cb864b2124db2567f 2024-11-17T22:42:15,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/bd761915fef04406914b93eac024b966 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/bd761915fef04406914b93eac024b966 2024-11-17T22:42:15,867 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/bd761915fef04406914b93eac024b966, entries=150, sequenceid=40, filesize=30.2 K 2024-11-17T22:42:15,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/faff33ae15054d819e657bea6fa65a27 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/faff33ae15054d819e657bea6fa65a27 2024-11-17T22:42:15,874 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/faff33ae15054d819e657bea6fa65a27, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T22:42:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/218f68fb9063448cb864b2124db2567f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/218f68fb9063448cb864b2124db2567f 2024-11-17T22:42:15,880 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/218f68fb9063448cb864b2124db2567f, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T22:42:15,881 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1e5636f3ed44c24ae3f10ff764ecff86 in 1400ms, sequenceid=40, compaction requested=false 2024-11-17T22:42:15,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:15,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:15,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-17T22:42:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-17T22:42:15,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-17T22:42:15,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4730 sec 2024-11-17T22:42:15,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.4810 sec 2024-11-17T22:42:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:16,790 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:16,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T22:42:16,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176ab396911b644caab1535771e7f9e6ea_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883334641/Put/seqid=0 2024-11-17T22:42:16,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742327_1503 (size=14594) 2024-11-17T22:42:16,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883396828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:16,844 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:16,848 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176ab396911b644caab1535771e7f9e6ea_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ab396911b644caab1535771e7f9e6ea_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:16,849 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/4ef80568c8154245869d1df20ab67438, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:16,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/4ef80568c8154245869d1df20ab67438 is 175, key is test_row_0/A:col10/1731883334641/Put/seqid=0 2024-11-17T22:42:16,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883396842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:16,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883396843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:16,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883396844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883396844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:16,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742328_1504 (size=39549) 2024-11-17T22:42:16,865 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/4ef80568c8154245869d1df20ab67438 2024-11-17T22:42:16,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/2935120329bc41ce8418c67acf671fbe is 50, key is test_row_0/B:col10/1731883334641/Put/seqid=0 2024-11-17T22:42:16,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742329_1505 (size=12001) 2024-11-17T22:42:16,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/2935120329bc41ce8418c67acf671fbe 2024-11-17T22:42:16,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/d64c2ecefcb6483387416e4cfaae9da4 is 50, key is test_row_0/C:col10/1731883334641/Put/seqid=0 2024-11-17T22:42:16,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883396947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:16,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883396957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883396957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883396957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883396957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:16,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742330_1506 (size=12001) 2024-11-17T22:42:16,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/d64c2ecefcb6483387416e4cfaae9da4 2024-11-17T22:42:16,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/4ef80568c8154245869d1df20ab67438 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4ef80568c8154245869d1df20ab67438 2024-11-17T22:42:17,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4ef80568c8154245869d1df20ab67438, entries=200, sequenceid=53, filesize=38.6 K 2024-11-17T22:42:17,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/2935120329bc41ce8418c67acf671fbe as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/2935120329bc41ce8418c67acf671fbe 2024-11-17T22:42:17,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/2935120329bc41ce8418c67acf671fbe, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T22:42:17,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/d64c2ecefcb6483387416e4cfaae9da4 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/d64c2ecefcb6483387416e4cfaae9da4 2024-11-17T22:42:17,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/d64c2ecefcb6483387416e4cfaae9da4, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T22:42:17,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 1e5636f3ed44c24ae3f10ff764ecff86 in 226ms, sequenceid=53, compaction requested=true 2024-11-17T22:42:17,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:17,016 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:17,017 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:17,017 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/A is initiating minor compaction (all files) 2024-11-17T22:42:17,017 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/A in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:17,017 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/7b60c22263554005b6650cb1c5ed756e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/bd761915fef04406914b93eac024b966, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4ef80568c8154245869d1df20ab67438] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=99.1 K 2024-11-17T22:42:17,017 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:17,018 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/7b60c22263554005b6650cb1c5ed756e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/bd761915fef04406914b93eac024b966, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4ef80568c8154245869d1df20ab67438] 2024-11-17T22:42:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:42:17,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:17,018 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b60c22263554005b6650cb1c5ed756e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731883333405 2024-11-17T22:42:17,018 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd761915fef04406914b93eac024b966, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731883333465 2024-11-17T22:42:17,019 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ef80568c8154245869d1df20ab67438, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731883334634 2024-11-17T22:42:17,019 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:17,021 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:17,021 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/B is initiating minor compaction (all files) 2024-11-17T22:42:17,021 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/B in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:17,021 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cd7476ef8998443c834d71e112b2b8c2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/faff33ae15054d819e657bea6fa65a27, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/2935120329bc41ce8418c67acf671fbe] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=35.2 K 2024-11-17T22:42:17,022 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting cd7476ef8998443c834d71e112b2b8c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731883333405 2024-11-17T22:42:17,022 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting faff33ae15054d819e657bea6fa65a27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731883333465 2024-11-17T22:42:17,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:42:17,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:17,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:17,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:17,023 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 2935120329bc41ce8418c67acf671fbe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731883334641 2024-11-17T22:42:17,047 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:17,059 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:17,059 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/d7a07b1ee8b84710b1244c2fd36d09f0 is 50, key is test_row_0/B:col10/1731883334641/Put/seqid=0 2024-11-17T22:42:17,070 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117c55f2d58de2c4e998421b3feeb6875c0_1e5636f3ed44c24ae3f10ff764ecff86 store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:17,072 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117c55f2d58de2c4e998421b3feeb6875c0_1e5636f3ed44c24ae3f10ff764ecff86, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:17,072 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c55f2d58de2c4e998421b3feeb6875c0_1e5636f3ed44c24ae3f10ff764ecff86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:17,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742331_1507 (size=12104) 2024-11-17T22:42:17,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742332_1508 (size=4469) 2024-11-17T22:42:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:17,161 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:17,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:42:17,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cfceb3bea3cf4e09898251a3f6fea0e2_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883336841/Put/seqid=0 2024-11-17T22:42:17,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883397173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883397173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883397175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883397176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883397178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742333_1509 (size=14594) 2024-11-17T22:42:17,197 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,202 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cfceb3bea3cf4e09898251a3f6fea0e2_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cfceb3bea3cf4e09898251a3f6fea0e2_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:17,202 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/5c170c1966b942bf9249d8be0498cca5, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:17,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/5c170c1966b942bf9249d8be0498cca5 is 175, key is test_row_0/A:col10/1731883336841/Put/seqid=0 2024-11-17T22:42:17,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742334_1510 (size=39549) 2024-11-17T22:42:17,235 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/5c170c1966b942bf9249d8be0498cca5 2024-11-17T22:42:17,246 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T22:42:17,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/5f5075a8ba684801bd92d2b971f594aa is 50, key is test_row_0/B:col10/1731883336841/Put/seqid=0 2024-11-17T22:42:17,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742335_1511 (size=12001) 2024-11-17T22:42:17,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/5f5075a8ba684801bd92d2b971f594aa 2024-11-17T22:42:17,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/2eaad9b393314736b5c46481d68e7149 is 50, key is test_row_0/C:col10/1731883336841/Put/seqid=0 2024-11-17T22:42:17,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883397288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883397288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883397288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883397288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883397289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742336_1512 (size=12001) 2024-11-17T22:42:17,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/2eaad9b393314736b5c46481d68e7149 2024-11-17T22:42:17,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/5c170c1966b942bf9249d8be0498cca5 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5c170c1966b942bf9249d8be0498cca5 2024-11-17T22:42:17,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5c170c1966b942bf9249d8be0498cca5, entries=200, sequenceid=78, filesize=38.6 K 2024-11-17T22:42:17,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/5f5075a8ba684801bd92d2b971f594aa as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5f5075a8ba684801bd92d2b971f594aa 2024-11-17T22:42:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5f5075a8ba684801bd92d2b971f594aa, entries=150, sequenceid=78, filesize=11.7 K 2024-11-17T22:42:17,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/2eaad9b393314736b5c46481d68e7149 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2eaad9b393314736b5c46481d68e7149 2024-11-17T22:42:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2eaad9b393314736b5c46481d68e7149, entries=150, sequenceid=78, filesize=11.7 K 2024-11-17T22:42:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1e5636f3ed44c24ae3f10ff764ecff86 in 211ms, sequenceid=78, compaction requested=true 2024-11-17T22:42:17,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:17,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:A, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:17,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T22:42:17,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:B, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:17,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-17T22:42:17,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:17,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-17T22:42:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,513 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:17,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:42:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,521 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/d7a07b1ee8b84710b1244c2fd36d09f0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/d7a07b1ee8b84710b1244c2fd36d09f0 2024-11-17T22:42:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,526 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/B of 1e5636f3ed44c24ae3f10ff764ecff86 into d7a07b1ee8b84710b1244c2fd36d09f0(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:17,526 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:17,526 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/B, priority=13, startTime=1731883337019; duration=0sec 2024-11-17T22:42:17,527 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-17T22:42:17,527 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:B 2024-11-17T22:42:17,527 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-17T22:42:17,527 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T22:42:17,528 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T22:42:17,528 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. because compaction request was cancelled 2024-11-17T22:42:17,528 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:A 2024-11-17T22:42:17,528 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:42:17,529 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:42:17,529 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/C is initiating minor compaction (all files) 2024-11-17T22:42:17,529 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/C in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:17,529 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/911d5ce95ff1401dabec9a58c76af4a4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/218f68fb9063448cb864b2124db2567f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/d64c2ecefcb6483387416e4cfaae9da4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2eaad9b393314736b5c46481d68e7149] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=46.9 K 2024-11-17T22:42:17,529 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 911d5ce95ff1401dabec9a58c76af4a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731883333405 2024-11-17T22:42:17,530 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 218f68fb9063448cb864b2124db2567f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731883333465 2024-11-17T22:42:17,530 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d64c2ecefcb6483387416e4cfaae9da4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731883334641 2024-11-17T22:42:17,530 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eaad9b393314736b5c46481d68e7149, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731883336822 2024-11-17T22:42:17,531 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#A#compaction#434 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:17,531 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/ac19d11a223a4c5cbf0f939123bb8d3b is 175, key is test_row_0/A:col10/1731883334641/Put/seqid=0 2024-11-17T22:42:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177675699bd9174255b95acd28b87b7893_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883337174/Put/seqid=0 2024-11-17T22:42:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,558 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#C#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:17,558 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/092e0c7ca5a342bca8655369141c4c68 is 50, key is test_row_0/C:col10/1731883336841/Put/seqid=0 2024-11-17T22:42:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T22:42:17,567 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:17,568 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:42:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-17T22:42:17,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T22:42:17,571 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:42:17,572 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:42:17,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:42:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883397589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883397599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742337_1513 (size=31058) 2024-11-17T22:42:17,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742338_1514 (size=12154) 2024-11-17T22:42:17,623 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:17,629 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177675699bd9174255b95acd28b87b7893_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177675699bd9174255b95acd28b87b7893_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:17,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883397615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883397618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883397619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,638 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/2f4cff1198624983aff42e37fd614ffe, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:17,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/2f4cff1198624983aff42e37fd614ffe is 175, key is test_row_0/A:col10/1731883337174/Put/seqid=0 2024-11-17T22:42:17,639 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/ac19d11a223a4c5cbf0f939123bb8d3b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ac19d11a223a4c5cbf0f939123bb8d3b 2024-11-17T22:42:17,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742339_1515 (size=12139) 2024-11-17T22:42:17,654 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/A of 1e5636f3ed44c24ae3f10ff764ecff86 into ac19d11a223a4c5cbf0f939123bb8d3b(size=30.3 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:17,655 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:17,655 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/A, priority=13, startTime=1731883337016; duration=0sec 2024-11-17T22:42:17,655 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T22:42:17,655 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:A 2024-11-17T22:42:17,655 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:B 2024-11-17T22:42:17,655 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-17T22:42:17,655 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T22:42:17,655 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T22:42:17,655 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. because compaction request was cancelled 2024-11-17T22:42:17,655 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:C 2024-11-17T22:42:17,655 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-17T22:42:17,656 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T22:42:17,656 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T22:42:17,656 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. because compaction request was cancelled 2024-11-17T22:42:17,656 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/092e0c7ca5a342bca8655369141c4c68 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/092e0c7ca5a342bca8655369141c4c68 2024-11-17T22:42:17,656 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:B 2024-11-17T22:42:17,664 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/C of 1e5636f3ed44c24ae3f10ff764ecff86 into 092e0c7ca5a342bca8655369141c4c68(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:17,664 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:17,664 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/C, priority=12, startTime=1731883337372; duration=0sec 2024-11-17T22:42:17,664 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:17,664 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:C 2024-11-17T22:42:17,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T22:42:17,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742340_1516 (size=30955) 2024-11-17T22:42:17,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883397727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883397727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T22:42:17,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:17,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:17,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:17,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:17,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:17,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883397736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883397737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883397737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T22:42:17,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T22:42:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:17,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:17,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883397936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883397937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883397947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883397947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883397948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T22:42:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:18,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:18,081 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/2f4cff1198624983aff42e37fd614ffe 2024-11-17T22:42:18,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/5b7eec51f75743e5b3c535d19fab2895 is 50, key is test_row_0/B:col10/1731883337174/Put/seqid=0 2024-11-17T22:42:18,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742341_1517 (size=12001) 2024-11-17T22:42:18,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/5b7eec51f75743e5b3c535d19fab2895 2024-11-17T22:42:18,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/595b872e26e445bf8cede5e8656edfe5 is 50, key is test_row_0/C:col10/1731883337174/Put/seqid=0 2024-11-17T22:42:18,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742342_1518 (size=12001) 2024-11-17T22:42:18,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/595b872e26e445bf8cede5e8656edfe5 2024-11-17T22:42:18,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/2f4cff1198624983aff42e37fd614ffe as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/2f4cff1198624983aff42e37fd614ffe 2024-11-17T22:42:18,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/2f4cff1198624983aff42e37fd614ffe, entries=150, sequenceid=90, filesize=30.2 K 2024-11-17T22:42:18,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/5b7eec51f75743e5b3c535d19fab2895 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5b7eec51f75743e5b3c535d19fab2895 2024-11-17T22:42:18,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5b7eec51f75743e5b3c535d19fab2895, entries=150, sequenceid=90, filesize=11.7 K 2024-11-17T22:42:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/595b872e26e445bf8cede5e8656edfe5 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/595b872e26e445bf8cede5e8656edfe5 2024-11-17T22:42:18,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/595b872e26e445bf8cede5e8656edfe5, entries=150, sequenceid=90, filesize=11.7 K 2024-11-17T22:42:18,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1e5636f3ed44c24ae3f10ff764ecff86 in 672ms, sequenceid=90, compaction requested=true 2024-11-17T22:42:18,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:18,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:42:18,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:18,185 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:18,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:42:18,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:18,185 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:18,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:18,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:18,186 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:18,186 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:18,186 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/B is initiating minor compaction (all files) 2024-11-17T22:42:18,186 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/A is initiating minor compaction (all files) 2024-11-17T22:42:18,186 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/A in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:18,186 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/B in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:18,186 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ac19d11a223a4c5cbf0f939123bb8d3b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5c170c1966b942bf9249d8be0498cca5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/2f4cff1198624983aff42e37fd614ffe] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=99.2 K 2024-11-17T22:42:18,186 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/d7a07b1ee8b84710b1244c2fd36d09f0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5f5075a8ba684801bd92d2b971f594aa, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5b7eec51f75743e5b3c535d19fab2895] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=35.3 K 2024-11-17T22:42:18,186 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:18,186 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ac19d11a223a4c5cbf0f939123bb8d3b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5c170c1966b942bf9249d8be0498cca5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/2f4cff1198624983aff42e37fd614ffe] 2024-11-17T22:42:18,186 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d7a07b1ee8b84710b1244c2fd36d09f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731883334641 2024-11-17T22:42:18,187 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac19d11a223a4c5cbf0f939123bb8d3b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731883334641 2024-11-17T22:42:18,187 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5075a8ba684801bd92d2b971f594aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731883336822 2024-11-17T22:42:18,187 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c170c1966b942bf9249d8be0498cca5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731883336822 2024-11-17T22:42:18,187 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b7eec51f75743e5b3c535d19fab2895, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731883337174 2024-11-17T22:42:18,187 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f4cff1198624983aff42e37fd614ffe, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731883337174 2024-11-17T22:42:18,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T22:42:18,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:18,196 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:42:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T22:42:18,206 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:18,226 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:18,226 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/02380a74d4a84677851ee603f49182ea is 50, key is test_row_0/B:col10/1731883337174/Put/seqid=0 2024-11-17T22:42:18,233 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411178502af2927ea4f62a1522d8d34b63690_1e5636f3ed44c24ae3f10ff764ecff86 store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:18,235 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411178502af2927ea4f62a1522d8d34b63690_1e5636f3ed44c24ae3f10ff764ecff86, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:18,235 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178502af2927ea4f62a1522d8d34b63690_1e5636f3ed44c24ae3f10ff764ecff86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:18,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:18,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117eb0b22418aa14499bc5e956ee21a9103_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883337604/Put/seqid=0 2024-11-17T22:42:18,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742343_1519 (size=12207) 2024-11-17T22:42:18,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883398260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883398260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883398261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883398261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883398262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742344_1520 (size=4469) 2024-11-17T22:42:18,312 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#A#compaction#443 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:18,313 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/37ebc698a09b420494b15326e114a9ed is 175, key is test_row_0/A:col10/1731883337174/Put/seqid=0 2024-11-17T22:42:18,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742345_1521 (size=12154) 2024-11-17T22:42:18,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:18,327 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117eb0b22418aa14499bc5e956ee21a9103_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eb0b22418aa14499bc5e956ee21a9103_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/5742c6e09dde44908b9d37bc8edc342e, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:18,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/5742c6e09dde44908b9d37bc8edc342e is 175, key is test_row_0/A:col10/1731883337604/Put/seqid=0 2024-11-17T22:42:18,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742346_1522 (size=31161) 2024-11-17T22:42:18,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742347_1523 (size=30955) 2024-11-17T22:42:18,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883398377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883398377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883398377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883398377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883398378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883398587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883398587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883398587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883398588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883398588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,668 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/02380a74d4a84677851ee603f49182ea as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/02380a74d4a84677851ee603f49182ea 2024-11-17T22:42:18,672 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/B of 1e5636f3ed44c24ae3f10ff764ecff86 into 02380a74d4a84677851ee603f49182ea(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:18,672 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:18,672 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/B, priority=13, startTime=1731883338185; duration=0sec 2024-11-17T22:42:18,672 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:18,672 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:B 2024-11-17T22:42:18,673 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-17T22:42:18,673 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T22:42:18,673 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T22:42:18,673 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. because compaction request was cancelled 2024-11-17T22:42:18,673 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:C 2024-11-17T22:42:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T22:42:18,772 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/5742c6e09dde44908b9d37bc8edc342e 2024-11-17T22:42:18,776 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/37ebc698a09b420494b15326e114a9ed as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/37ebc698a09b420494b15326e114a9ed 2024-11-17T22:42:18,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/11cd8dc344c44d2b860057903165284b is 50, key is test_row_0/B:col10/1731883337604/Put/seqid=0 2024-11-17T22:42:18,783 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/A of 1e5636f3ed44c24ae3f10ff764ecff86 into 37ebc698a09b420494b15326e114a9ed(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:18,783 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:18,783 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/A, priority=13, startTime=1731883338185; duration=0sec 2024-11-17T22:42:18,783 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:18,783 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:A 2024-11-17T22:42:18,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742348_1524 (size=12001) 2024-11-17T22:42:18,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883398896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883398897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883398897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883398897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883398897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:19,215 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/11cd8dc344c44d2b860057903165284b 2024-11-17T22:42:19,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/b1f28c31a2734e21ae067ffdcbccc51a is 50, key is test_row_0/C:col10/1731883337604/Put/seqid=0 2024-11-17T22:42:19,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742349_1525 (size=12001) 2024-11-17T22:42:19,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883399407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:19,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883399407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883399407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883399408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883399408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:19,682 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/b1f28c31a2734e21ae067ffdcbccc51a 2024-11-17T22:42:19,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/5742c6e09dde44908b9d37bc8edc342e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5742c6e09dde44908b9d37bc8edc342e 2024-11-17T22:42:19,688 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5742c6e09dde44908b9d37bc8edc342e, entries=150, sequenceid=118, filesize=30.2 K 2024-11-17T22:42:19,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/11cd8dc344c44d2b860057903165284b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/11cd8dc344c44d2b860057903165284b 2024-11-17T22:42:19,691 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/11cd8dc344c44d2b860057903165284b, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T22:42:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/b1f28c31a2734e21ae067ffdcbccc51a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b1f28c31a2734e21ae067ffdcbccc51a 2024-11-17T22:42:19,694 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b1f28c31a2734e21ae067ffdcbccc51a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T22:42:19,695 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1e5636f3ed44c24ae3f10ff764ecff86 in 1499ms, sequenceid=118, compaction requested=true 2024-11-17T22:42:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-17T22:42:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-17T22:42:19,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-17T22:42:19,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1250 sec 2024-11-17T22:42:19,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.1320 sec 2024-11-17T22:42:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T22:42:19,727 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:19,727 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:42:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-17T22:42:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T22:42:19,729 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:42:19,730 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:42:19,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:42:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T22:42:19,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T22:42:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:19,882 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:42:19,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b593bee5961f486c9b0341cb4ce888d2_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883338260/Put/seqid=0 2024-11-17T22:42:19,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742350_1526 (size=12154) 2024-11-17T22:42:19,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:19,897 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b593bee5961f486c9b0341cb4ce888d2_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b593bee5961f486c9b0341cb4ce888d2_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:19,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/8d96c693ce2e450cb81b6c8667f429fd, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:19,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/8d96c693ce2e450cb81b6c8667f429fd is 175, key is test_row_0/A:col10/1731883338260/Put/seqid=0 2024-11-17T22:42:19,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742351_1527 (size=30955) 2024-11-17T22:42:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T22:42:20,302 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/8d96c693ce2e450cb81b6c8667f429fd 2024-11-17T22:42:20,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/9df39d3682af42fa8c8a9ef35045e1c8 is 50, key is test_row_0/B:col10/1731883338260/Put/seqid=0 2024-11-17T22:42:20,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742352_1528 (size=12001) 2024-11-17T22:42:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T22:42:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:20,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883400459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883400458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883400460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883400461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883400467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883400577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883400577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883400577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883400577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883400577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,721 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/9df39d3682af42fa8c8a9ef35045e1c8 2024-11-17T22:42:20,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/ef7f95a80ec34e8eafa74a3d4bf16211 is 50, key is test_row_0/C:col10/1731883338260/Put/seqid=0 2024-11-17T22:42:20,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742353_1529 (size=12001) 2024-11-17T22:42:20,756 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/ef7f95a80ec34e8eafa74a3d4bf16211 2024-11-17T22:42:20,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/8d96c693ce2e450cb81b6c8667f429fd as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/8d96c693ce2e450cb81b6c8667f429fd 2024-11-17T22:42:20,765 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/8d96c693ce2e450cb81b6c8667f429fd, entries=150, sequenceid=131, filesize=30.2 K 2024-11-17T22:42:20,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/9df39d3682af42fa8c8a9ef35045e1c8 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/9df39d3682af42fa8c8a9ef35045e1c8 2024-11-17T22:42:20,770 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/9df39d3682af42fa8c8a9ef35045e1c8, entries=150, sequenceid=131, filesize=11.7 K 2024-11-17T22:42:20,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/ef7f95a80ec34e8eafa74a3d4bf16211 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/ef7f95a80ec34e8eafa74a3d4bf16211 2024-11-17T22:42:20,774 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/ef7f95a80ec34e8eafa74a3d4bf16211, entries=150, sequenceid=131, filesize=11.7 K 2024-11-17T22:42:20,775 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1e5636f3ed44c24ae3f10ff764ecff86 in 893ms, sequenceid=131, compaction requested=true 2024-11-17T22:42:20,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:20,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:20,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-17T22:42:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-17T22:42:20,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-17T22:42:20,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0460 sec 2024-11-17T22:42:20,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.0520 sec 2024-11-17T22:42:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:20,789 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:20,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T22:42:20,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117adbc6124d4a2455db4134d528715149b_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883340444/Put/seqid=0 2024-11-17T22:42:20,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883400791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883400792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742354_1530 (size=14794) 2024-11-17T22:42:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883400795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883400797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883400797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T22:42:20,867 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:20,867 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:42:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-17T22:42:20,868 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:42:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T22:42:20,869 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:42:20,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:42:20,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883400907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883400907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883400907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883400907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883400907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T22:42:21,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T22:42:21,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:21,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883401117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883401117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883401118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883401118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883401118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T22:42:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T22:42:21,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:21,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117adbc6124d4a2455db4134d528715149b_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117adbc6124d4a2455db4134d528715149b_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:21,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/63fca6d123bd4748ab12fa732238f393, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:21,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/63fca6d123bd4748ab12fa732238f393 is 175, key is test_row_0/A:col10/1731883340444/Put/seqid=0 2024-11-17T22:42:21,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742355_1531 (size=39749) 2024-11-17T22:42:21,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T22:42:21,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:21,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883401427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883401427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883401427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883401427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883401427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T22:42:21,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:21,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T22:42:21,623 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/63fca6d123bd4748ab12fa732238f393 2024-11-17T22:42:21,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/d57697627ab64018b2cc49471baa9714 is 50, key is test_row_0/B:col10/1731883340444/Put/seqid=0 2024-11-17T22:42:21,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T22:42:21,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:21,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742356_1532 (size=12151) 2024-11-17T22:42:21,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T22:42:21,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:21,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T22:42:21,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:21,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:21,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:21,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883401937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883401937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883401937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883401937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883401937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T22:42:22,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/d57697627ab64018b2cc49471baa9714 2024-11-17T22:42:22,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/2116da584e3541a48ac9c3119a37ac6a is 50, key is test_row_0/C:col10/1731883340444/Put/seqid=0 2024-11-17T22:42:22,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742357_1533 (size=12151) 2024-11-17T22:42:22,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T22:42:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:22,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:22,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T22:42:22,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:22,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:22,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:22,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:22,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T22:42:22,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:22,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:22,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:22,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:22,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:22,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/2116da584e3541a48ac9c3119a37ac6a 2024-11-17T22:42:22,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/63fca6d123bd4748ab12fa732238f393 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/63fca6d123bd4748ab12fa732238f393 2024-11-17T22:42:22,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/63fca6d123bd4748ab12fa732238f393, entries=200, sequenceid=157, filesize=38.8 K 2024-11-17T22:42:22,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/d57697627ab64018b2cc49471baa9714 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/d57697627ab64018b2cc49471baa9714 2024-11-17T22:42:22,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/d57697627ab64018b2cc49471baa9714, entries=150, sequenceid=157, filesize=11.9 K 2024-11-17T22:42:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/2116da584e3541a48ac9c3119a37ac6a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2116da584e3541a48ac9c3119a37ac6a 2024-11-17T22:42:22,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2116da584e3541a48ac9c3119a37ac6a, entries=150, sequenceid=157, filesize=11.9 K 2024-11-17T22:42:22,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1e5636f3ed44c24ae3f10ff764ecff86 in 1672ms, sequenceid=157, compaction requested=true 2024-11-17T22:42:22,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:22,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:42:22,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:22,461 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:42:22,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:42:22,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:22,461 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:42:22,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:22,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:22,462 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:42:22,462 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/B is initiating minor compaction (all files) 2024-11-17T22:42:22,462 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/B in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:22,462 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/02380a74d4a84677851ee603f49182ea, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/11cd8dc344c44d2b860057903165284b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/9df39d3682af42fa8c8a9ef35045e1c8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/d57697627ab64018b2cc49471baa9714] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=47.2 K 2024-11-17T22:42:22,463 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 02380a74d4a84677851ee603f49182ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731883337174 2024-11-17T22:42:22,463 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:42:22,463 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/A is initiating minor compaction (all files) 2024-11-17T22:42:22,463 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/A in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:22,463 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/37ebc698a09b420494b15326e114a9ed, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5742c6e09dde44908b9d37bc8edc342e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/8d96c693ce2e450cb81b6c8667f429fd, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/63fca6d123bd4748ab12fa732238f393] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=129.7 K 2024-11-17T22:42:22,463 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:22,463 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/37ebc698a09b420494b15326e114a9ed, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5742c6e09dde44908b9d37bc8edc342e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/8d96c693ce2e450cb81b6c8667f429fd, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/63fca6d123bd4748ab12fa732238f393] 2024-11-17T22:42:22,473 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 11cd8dc344c44d2b860057903165284b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731883337597 2024-11-17T22:42:22,473 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37ebc698a09b420494b15326e114a9ed, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731883337174 2024-11-17T22:42:22,473 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 9df39d3682af42fa8c8a9ef35045e1c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731883338250 2024-11-17T22:42:22,473 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5742c6e09dde44908b9d37bc8edc342e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731883337597 2024-11-17T22:42:22,474 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d96c693ce2e450cb81b6c8667f429fd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731883338250 2024-11-17T22:42:22,474 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting d57697627ab64018b2cc49471baa9714, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883340444 2024-11-17T22:42:22,474 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63fca6d123bd4748ab12fa732238f393, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883340444 2024-11-17T22:42:22,493 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:22,495 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#B#compaction#454 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:22,495 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117c25020d55aad43feba825ee59e9d530f_1e5636f3ed44c24ae3f10ff764ecff86 store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:22,496 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/5ef8f4b3fcfb49c19b76b96d230dd2fa is 50, key is test_row_0/B:col10/1731883340444/Put/seqid=0 2024-11-17T22:42:22,498 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117c25020d55aad43feba825ee59e9d530f_1e5636f3ed44c24ae3f10ff764ecff86, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:22,498 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c25020d55aad43feba825ee59e9d530f_1e5636f3ed44c24ae3f10ff764ecff86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:22,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742358_1534 (size=12493) 2024-11-17T22:42:22,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742359_1535 (size=4469) 2024-11-17T22:42:22,525 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#A#compaction#455 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:22,526 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/ab8eebde61dc42ca864a1ff25e037c72 is 175, key is test_row_0/A:col10/1731883340444/Put/seqid=0 2024-11-17T22:42:22,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742360_1536 (size=31447) 2024-11-17T22:42:22,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T22:42:22,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:22,548 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T22:42:22,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117640b9e8c893f46218412870c7c84bfe2_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883340795/Put/seqid=0 2024-11-17T22:42:22,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742361_1537 (size=12304) 2024-11-17T22:42:22,915 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/5ef8f4b3fcfb49c19b76b96d230dd2fa as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5ef8f4b3fcfb49c19b76b96d230dd2fa 2024-11-17T22:42:22,919 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/B of 1e5636f3ed44c24ae3f10ff764ecff86 into 5ef8f4b3fcfb49c19b76b96d230dd2fa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:22,919 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:22,919 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/B, priority=12, startTime=1731883342461; duration=0sec 2024-11-17T22:42:22,919 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:22,919 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:B 2024-11-17T22:42:22,919 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T22:42:22,920 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60293 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T22:42:22,920 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/C is initiating minor compaction (all files) 2024-11-17T22:42:22,920 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/C in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:22,920 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/092e0c7ca5a342bca8655369141c4c68, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/595b872e26e445bf8cede5e8656edfe5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b1f28c31a2734e21ae067ffdcbccc51a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/ef7f95a80ec34e8eafa74a3d4bf16211, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2116da584e3541a48ac9c3119a37ac6a] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=58.9 K 2024-11-17T22:42:22,920 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 092e0c7ca5a342bca8655369141c4c68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731883336822 2024-11-17T22:42:22,921 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 595b872e26e445bf8cede5e8656edfe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731883337174 2024-11-17T22:42:22,921 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f28c31a2734e21ae067ffdcbccc51a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731883337597 2024-11-17T22:42:22,921 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting ef7f95a80ec34e8eafa74a3d4bf16211, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731883338250 2024-11-17T22:42:22,921 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 2116da584e3541a48ac9c3119a37ac6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883340444 2024-11-17T22:42:22,929 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#C#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:22,930 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/9ce430b5c6b44b859fbbc11b793acecd is 50, key is test_row_0/C:col10/1731883340444/Put/seqid=0 2024-11-17T22:42:22,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742362_1538 (size=12459) 2024-11-17T22:42:22,942 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/9ce430b5c6b44b859fbbc11b793acecd as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9ce430b5c6b44b859fbbc11b793acecd 2024-11-17T22:42:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:22,957 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/ab8eebde61dc42ca864a1ff25e037c72 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ab8eebde61dc42ca864a1ff25e037c72 2024-11-17T22:42:22,957 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/C of 1e5636f3ed44c24ae3f10ff764ecff86 into 9ce430b5c6b44b859fbbc11b793acecd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:22,957 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:22,957 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/C, priority=11, startTime=1731883342461; duration=0sec 2024-11-17T22:42:22,957 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:22,957 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:C 2024-11-17T22:42:22,963 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/A of 1e5636f3ed44c24ae3f10ff764ecff86 into ab8eebde61dc42ca864a1ff25e037c72(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:22,963 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:22,963 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/A, priority=12, startTime=1731883342461; duration=0sec 2024-11-17T22:42:22,963 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:22,963 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:A 2024-11-17T22:42:22,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:22,967 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117640b9e8c893f46218412870c7c84bfe2_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117640b9e8c893f46218412870c7c84bfe2_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:22,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/faf3923d4df44fe096e3f5b39f5af05b, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:22,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/faf3923d4df44fe096e3f5b39f5af05b is 175, key is test_row_0/A:col10/1731883340795/Put/seqid=0 2024-11-17T22:42:22,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883402983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:22,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883402984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:22,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883402985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:22,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883402985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:22,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883402987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T22:42:23,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742363_1539 (size=31105) 2024-11-17T22:42:23,027 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/faf3923d4df44fe096e3f5b39f5af05b 2024-11-17T22:42:23,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/5e2ba17661d14094bec00c33a78baa80 is 50, key is test_row_0/B:col10/1731883340795/Put/seqid=0 2024-11-17T22:42:23,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742364_1540 (size=12151) 2024-11-17T22:42:23,063 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/5e2ba17661d14094bec00c33a78baa80 2024-11-17T22:42:23,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/7656c305d5ab4b9c9f196d23957d9de2 is 50, key is test_row_0/C:col10/1731883340795/Put/seqid=0 2024-11-17T22:42:23,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742365_1541 (size=12151) 2024-11-17T22:42:23,092 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/7656c305d5ab4b9c9f196d23957d9de2 2024-11-17T22:42:23,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/faf3923d4df44fe096e3f5b39f5af05b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/faf3923d4df44fe096e3f5b39f5af05b 2024-11-17T22:42:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883403102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883403103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883403103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883403103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883403103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,110 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/faf3923d4df44fe096e3f5b39f5af05b, entries=150, sequenceid=167, filesize=30.4 K 2024-11-17T22:42:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/5e2ba17661d14094bec00c33a78baa80 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5e2ba17661d14094bec00c33a78baa80 2024-11-17T22:42:23,120 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5e2ba17661d14094bec00c33a78baa80, entries=150, sequenceid=167, filesize=11.9 K 2024-11-17T22:42:23,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/7656c305d5ab4b9c9f196d23957d9de2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7656c305d5ab4b9c9f196d23957d9de2 2024-11-17T22:42:23,124 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7656c305d5ab4b9c9f196d23957d9de2, entries=150, sequenceid=167, filesize=11.9 K 2024-11-17T22:42:23,125 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 1e5636f3ed44c24ae3f10ff764ecff86 in 577ms, sequenceid=167, compaction requested=false 2024-11-17T22:42:23,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:23,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:23,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-17T22:42:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-17T22:42:23,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-17T22:42:23,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2580 sec 2024-11-17T22:42:23,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.2650 sec 2024-11-17T22:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:23,318 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:23,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-17T22:42:23,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883403320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883403322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883403322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a2131b9c066e46bfa864e2b009bfa071_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883343317/Put/seqid=0 2024-11-17T22:42:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883403323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883403324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742366_1542 (size=14794) 2024-11-17T22:42:23,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883403427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883403427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883403437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883403437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883403437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883403637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883403637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883403647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883403647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883403647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,735 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:23,738 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a2131b9c066e46bfa864e2b009bfa071_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a2131b9c066e46bfa864e2b009bfa071_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:23,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/f8edd65f7bee43a1a81054e0a31ac0fc, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:23,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/f8edd65f7bee43a1a81054e0a31ac0fc is 175, key is test_row_0/A:col10/1731883343317/Put/seqid=0 2024-11-17T22:42:23,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742367_1543 (size=39749) 2024-11-17T22:42:23,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883403946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883403947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883403957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883403957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883403957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:24,143 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/f8edd65f7bee43a1a81054e0a31ac0fc 2024-11-17T22:42:24,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/b75d4c615d0c4ecc926aa60d73a20185 is 50, key is test_row_0/B:col10/1731883343317/Put/seqid=0 2024-11-17T22:42:24,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742368_1544 (size=12151) 2024-11-17T22:42:24,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883404457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:24,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:24,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:24,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883404457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:24,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:24,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:24,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883404467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883404467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883404468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:24,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/b75d4c615d0c4ecc926aa60d73a20185 2024-11-17T22:42:24,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/6478c1ff16f04b5ba0ddc0add353f71d is 50, key is test_row_0/C:col10/1731883343317/Put/seqid=0 2024-11-17T22:42:24,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742369_1545 (size=12151) 2024-11-17T22:42:24,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/6478c1ff16f04b5ba0ddc0add353f71d 2024-11-17T22:42:24,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/f8edd65f7bee43a1a81054e0a31ac0fc as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f8edd65f7bee43a1a81054e0a31ac0fc 2024-11-17T22:42:24,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f8edd65f7bee43a1a81054e0a31ac0fc, entries=200, sequenceid=198, filesize=38.8 K 2024-11-17T22:42:24,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/b75d4c615d0c4ecc926aa60d73a20185 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b75d4c615d0c4ecc926aa60d73a20185 2024-11-17T22:42:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b75d4c615d0c4ecc926aa60d73a20185, entries=150, sequenceid=198, filesize=11.9 K 2024-11-17T22:42:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/6478c1ff16f04b5ba0ddc0add353f71d as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6478c1ff16f04b5ba0ddc0add353f71d 2024-11-17T22:42:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6478c1ff16f04b5ba0ddc0add353f71d, entries=150, sequenceid=198, filesize=11.9 K 2024-11-17T22:42:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 1e5636f3ed44c24ae3f10ff764ecff86 in 1273ms, sequenceid=198, compaction requested=true 2024-11-17T22:42:24,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:24,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:42:24,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:24,591 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:24,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:42:24,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:24,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:24,591 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,593 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,593 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/B is initiating minor compaction (all files) 2024-11-17T22:42:24,593 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/B in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:24,593 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5ef8f4b3fcfb49c19b76b96d230dd2fa, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5e2ba17661d14094bec00c33a78baa80, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b75d4c615d0c4ecc926aa60d73a20185] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=35.9 K 2024-11-17T22:42:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,593 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:24,593 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/A is initiating minor compaction (all files) 2024-11-17T22:42:24,593 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/A in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:24,593 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ab8eebde61dc42ca864a1ff25e037c72, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/faf3923d4df44fe096e3f5b39f5af05b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f8edd65f7bee43a1a81054e0a31ac0fc] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=99.9 K 2024-11-17T22:42:24,593 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:24,593 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ab8eebde61dc42ca864a1ff25e037c72, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/faf3923d4df44fe096e3f5b39f5af05b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f8edd65f7bee43a1a81054e0a31ac0fc] 2024-11-17T22:42:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,594 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ef8f4b3fcfb49c19b76b96d230dd2fa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883340444 2024-11-17T22:42:24,594 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab8eebde61dc42ca864a1ff25e037c72, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883340444 2024-11-17T22:42:24,594 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e2ba17661d14094bec00c33a78baa80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731883340789 2024-11-17T22:42:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,595 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting faf3923d4df44fe096e3f5b39f5af05b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731883340789 2024-11-17T22:42:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,595 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b75d4c615d0c4ecc926aa60d73a20185, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731883342983 2024-11-17T22:42:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,596 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8edd65f7bee43a1a81054e0a31ac0fc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731883342983 2024-11-17T22:42:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,603 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,606 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#B#compaction#463 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,606 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/ddd1316d4ae44574923a9db884c8df37 is 50, key is test_row_0/B:col10/1731883343317/Put/seqid=0 2024-11-17T22:42:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,617 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117385a9ed428e74d44a7c16639a7a0927d_1e5636f3ed44c24ae3f10ff764ecff86 store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742370_1546 (size=12595) 2024-11-17T22:42:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,623 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117385a9ed428e74d44a7c16639a7a0927d_1e5636f3ed44c24ae3f10ff764ecff86, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:24,623 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117385a9ed428e74d44a7c16639a7a0927d_1e5636f3ed44c24ae3f10ff764ecff86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,624 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/ddd1316d4ae44574923a9db884c8df37 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/ddd1316d4ae44574923a9db884c8df37 2024-11-17T22:42:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,629 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/B of 1e5636f3ed44c24ae3f10ff764ecff86 into ddd1316d4ae44574923a9db884c8df37(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,629 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:24,629 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/B, priority=13, startTime=1731883344591; duration=0sec 2024-11-17T22:42:24,629 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:24,629 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:B 2024-11-17T22:42:24,629 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,631 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:24,631 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/C is initiating minor compaction (all files) 2024-11-17T22:42:24,631 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/C in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,631 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9ce430b5c6b44b859fbbc11b793acecd, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7656c305d5ab4b9c9f196d23957d9de2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6478c1ff16f04b5ba0ddc0add353f71d] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=35.9 K 2024-11-17T22:42:24,631 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ce430b5c6b44b859fbbc11b793acecd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731883340444 2024-11-17T22:42:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,632 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 7656c305d5ab4b9c9f196d23957d9de2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731883340789 2024-11-17T22:42:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,632 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 6478c1ff16f04b5ba0ddc0add353f71d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731883342983 2024-11-17T22:42:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742371_1547 (size=4469) 2024-11-17T22:42:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,657 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#C#compaction#465 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:24,658 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/6a047a2702b24276a8702b6e85ba2e3c is 50, key is test_row_0/C:col10/1731883343317/Put/seqid=0 2024-11-17T22:42:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742372_1548 (size=12561) 2024-11-17T22:42:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T22:42:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,027 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,027 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:42:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-17T22:42:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T22:42:25,032 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:42:25,033 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:42:25,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:42:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,051 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#A#compaction#464 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,052 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/5cbdc28c0e274c119426a5c8c45aa93a is 175, key is test_row_0/A:col10/1731883343317/Put/seqid=0 2024-11-17T22:42:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742373_1549 (size=31549) 2024-11-17T22:42:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,066 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/6a047a2702b24276a8702b6e85ba2e3c as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6a047a2702b24276a8702b6e85ba2e3c 2024-11-17T22:42:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,069 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/C of 1e5636f3ed44c24ae3f10ff764ecff86 into 6a047a2702b24276a8702b6e85ba2e3c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:25,069 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:25,069 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/C, priority=13, startTime=1731883344591; duration=0sec 2024-11-17T22:42:25,069 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:25,069 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:C 2024-11-17T22:42:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T22:42:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T22:42:25,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:25,186 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T22:42:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b2736cd3c36e4e71b19e0b8acb98a1b0_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_1/A:col10/1731883343321/Put/seqid=0 2024-11-17T22:42:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742374_1550 (size=9814) 2024-11-17T22:42:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,237 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b2736cd3c36e4e71b19e0b8acb98a1b0_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b2736cd3c36e4e71b19e0b8acb98a1b0_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/fcf96ad2280641bb952945342a3caa31, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/fcf96ad2280641bb952945342a3caa31 is 175, key is test_row_1/A:col10/1731883343321/Put/seqid=0 2024-11-17T22:42:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742375_1551 (size=22461) 2024-11-17T22:42:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,260 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/fcf96ad2280641bb952945342a3caa31 2024-11-17T22:42:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/89970e6c93184de6bd16907ff3be8113 is 50, key is test_row_1/B:col10/1731883343321/Put/seqid=0 2024-11-17T22:42:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742376_1552 (size=9757) 2024-11-17T22:42:25,288 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/89970e6c93184de6bd16907ff3be8113 2024-11-17T22:42:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/9ddbc3a6c07a42d0a267c1dd5659935b is 50, key is test_row_1/C:col10/1731883343321/Put/seqid=0 2024-11-17T22:42:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742377_1553 (size=9757) 2024-11-17T22:42:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,337 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/9ddbc3a6c07a42d0a267c1dd5659935b 2024-11-17T22:42:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/fcf96ad2280641bb952945342a3caa31 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/fcf96ad2280641bb952945342a3caa31 2024-11-17T22:42:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T22:42:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,348 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/fcf96ad2280641bb952945342a3caa31, entries=100, sequenceid=208, filesize=21.9 K 2024-11-17T22:42:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/89970e6c93184de6bd16907ff3be8113 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/89970e6c93184de6bd16907ff3be8113 2024-11-17T22:42:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,354 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/89970e6c93184de6bd16907ff3be8113, entries=100, sequenceid=208, filesize=9.5 K 2024-11-17T22:42:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/9ddbc3a6c07a42d0a267c1dd5659935b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9ddbc3a6c07a42d0a267c1dd5659935b 2024-11-17T22:42:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,359 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9ddbc3a6c07a42d0a267c1dd5659935b, entries=100, sequenceid=208, filesize=9.5 K 2024-11-17T22:42:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,360 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1e5636f3ed44c24ae3f10ff764ecff86 in 173ms, sequenceid=208, compaction requested=false 2024-11-17T22:42:25,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:25,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-17T22:42:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-17T22:42:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-17T22:42:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 328 msec 2024-11-17T22:42:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 336 msec 2024-11-17T22:42:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,469 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/5cbdc28c0e274c119426a5c8c45aa93a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5cbdc28c0e274c119426a5c8c45aa93a 2024-11-17T22:42:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,473 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/A of 1e5636f3ed44c24ae3f10ff764ecff86 into 5cbdc28c0e274c119426a5c8c45aa93a(size=30.8 K), total size for store is 52.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:25,473 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,473 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/A, priority=13, startTime=1731883344591; duration=0sec 2024-11-17T22:42:25,474 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:25,474 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:A 2024-11-17T22:42:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:25,505 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:42:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e060b8417e774c37992331b2e6e34916_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883345503/Put/seqid=0 2024-11-17T22:42:25,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742379_1555 (size=24758) 2024-11-17T22:42:25,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883405583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883405583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883405583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883405584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883405584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T22:42:25,657 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:25,658 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:42:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-17T22:42:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T22:42:25,659 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:42:25,660 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:42:25,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:42:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883405696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883405697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883405697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883405697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883405697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T22:42:25,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T22:42:25,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:25,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:25,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:25,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:25,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:25,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883405907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883405907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883405907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883405907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883405907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:25,959 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:25,963 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e060b8417e774c37992331b2e6e34916_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e060b8417e774c37992331b2e6e34916_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:25,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T22:42:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:25,963 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/926671b625ed49d49b7629e4d2d67fe7, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:25,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:25,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/926671b625ed49d49b7629e4d2d67fe7 is 175, key is test_row_0/A:col10/1731883345503/Put/seqid=0 2024-11-17T22:42:25,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742378_1554 (size=74395) 2024-11-17T22:42:25,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T22:42:26,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T22:42:26,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:26,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883406217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883406217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:26,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883406217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883406217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883406217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:26,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T22:42:26,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T22:42:26,368 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/926671b625ed49d49b7629e4d2d67fe7 2024-11-17T22:42:26,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/0f28738c7f1041ea908fe0406b6b40f1 is 50, key is test_row_0/B:col10/1731883345503/Put/seqid=0 2024-11-17T22:42:26,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742380_1556 (size=12151) 2024-11-17T22:42:26,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T22:42:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T22:42:26,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:26,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T22:42:26,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:26,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883406727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:26,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883406728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883406728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883406728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883406728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:26,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/0f28738c7f1041ea908fe0406b6b40f1 2024-11-17T22:42:26,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/7465dbc629f6461fa4bbab36f3dae8ad is 50, key is test_row_0/C:col10/1731883345503/Put/seqid=0 2024-11-17T22:42:26,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742381_1557 (size=12151) 2024-11-17T22:42:26,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/7465dbc629f6461fa4bbab36f3dae8ad 2024-11-17T22:42:26,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/926671b625ed49d49b7629e4d2d67fe7 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/926671b625ed49d49b7629e4d2d67fe7 2024-11-17T22:42:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T22:42:26,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/926671b625ed49d49b7629e4d2d67fe7, entries=400, sequenceid=220, filesize=72.7 K 2024-11-17T22:42:26,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/0f28738c7f1041ea908fe0406b6b40f1 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/0f28738c7f1041ea908fe0406b6b40f1 2024-11-17T22:42:26,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/0f28738c7f1041ea908fe0406b6b40f1, entries=150, sequenceid=220, filesize=11.9 K 2024-11-17T22:42:26,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/7465dbc629f6461fa4bbab36f3dae8ad as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7465dbc629f6461fa4bbab36f3dae8ad 2024-11-17T22:42:26,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7465dbc629f6461fa4bbab36f3dae8ad, entries=150, sequenceid=220, filesize=11.9 K 2024-11-17T22:42:26,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1e5636f3ed44c24ae3f10ff764ecff86 in 1302ms, sequenceid=220, compaction requested=true 2024-11-17T22:42:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:26,807 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:42:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:42:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:26,807 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:26,808 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:26,808 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:26,808 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/A is initiating minor compaction (all files) 2024-11-17T22:42:26,808 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/B is initiating minor compaction (all files) 2024-11-17T22:42:26,808 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/B in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,808 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/A in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,808 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/ddd1316d4ae44574923a9db884c8df37, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/89970e6c93184de6bd16907ff3be8113, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/0f28738c7f1041ea908fe0406b6b40f1] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=33.7 K 2024-11-17T22:42:26,808 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5cbdc28c0e274c119426a5c8c45aa93a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/fcf96ad2280641bb952945342a3caa31, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/926671b625ed49d49b7629e4d2d67fe7] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=125.4 K 2024-11-17T22:42:26,808 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,808 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5cbdc28c0e274c119426a5c8c45aa93a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/fcf96ad2280641bb952945342a3caa31, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/926671b625ed49d49b7629e4d2d67fe7] 2024-11-17T22:42:26,808 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting ddd1316d4ae44574923a9db884c8df37, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731883342983 2024-11-17T22:42:26,808 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cbdc28c0e274c119426a5c8c45aa93a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731883342983 2024-11-17T22:42:26,809 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 89970e6c93184de6bd16907ff3be8113, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731883343321 2024-11-17T22:42:26,809 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf96ad2280641bb952945342a3caa31, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731883343321 2024-11-17T22:42:26,810 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f28738c7f1041ea908fe0406b6b40f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731883345493 2024-11-17T22:42:26,810 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 926671b625ed49d49b7629e4d2d67fe7, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731883345488 2024-11-17T22:42:26,814 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:26,817 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#B#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:26,817 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117937544e0786e48b996ce8e3b21862f99_1e5636f3ed44c24ae3f10ff764ecff86 store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:26,818 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/6beacd2285d34fd6ba629e79f6ece809 is 50, key is test_row_0/B:col10/1731883345503/Put/seqid=0 2024-11-17T22:42:26,821 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117937544e0786e48b996ce8e3b21862f99_1e5636f3ed44c24ae3f10ff764ecff86, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:26,821 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117937544e0786e48b996ce8e3b21862f99_1e5636f3ed44c24ae3f10ff764ecff86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:26,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742382_1558 (size=12697) 2024-11-17T22:42:26,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742383_1559 (size=4469) 2024-11-17T22:42:26,830 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#A#compaction#472 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:26,831 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/c19c6e18a4e148a4baf2ffeeaf556769 is 175, key is test_row_0/A:col10/1731883345503/Put/seqid=0 2024-11-17T22:42:26,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742384_1560 (size=31651) 2024-11-17T22:42:26,846 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/c19c6e18a4e148a4baf2ffeeaf556769 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c19c6e18a4e148a4baf2ffeeaf556769 2024-11-17T22:42:26,853 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/A of 1e5636f3ed44c24ae3f10ff764ecff86 into c19c6e18a4e148a4baf2ffeeaf556769(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:26,853 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:26,853 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/A, priority=13, startTime=1731883346807; duration=0sec 2024-11-17T22:42:26,853 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:26,853 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:A 2024-11-17T22:42:26,853 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:26,855 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:26,855 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/C is initiating minor compaction (all files) 2024-11-17T22:42:26,855 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/C in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,855 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6a047a2702b24276a8702b6e85ba2e3c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9ddbc3a6c07a42d0a267c1dd5659935b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7465dbc629f6461fa4bbab36f3dae8ad] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=33.7 K 2024-11-17T22:42:26,855 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a047a2702b24276a8702b6e85ba2e3c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731883342983 2024-11-17T22:42:26,856 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ddbc3a6c07a42d0a267c1dd5659935b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731883343321 2024-11-17T22:42:26,856 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7465dbc629f6461fa4bbab36f3dae8ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731883345493 2024-11-17T22:42:26,879 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#C#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:26,879 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/6f016f5aa6c04a30825f815f2e791ff2 is 50, key is test_row_0/C:col10/1731883345503/Put/seqid=0 2024-11-17T22:42:26,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T22:42:26,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:26,882 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:42:26,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111765307476773d4cb9b8cab0bdc012743d_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883345547/Put/seqid=0 2024-11-17T22:42:26,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742385_1561 (size=12663) 2024-11-17T22:42:26,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742386_1562 (size=12304) 2024-11-17T22:42:26,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:26,931 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111765307476773d4cb9b8cab0bdc012743d_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111765307476773d4cb9b8cab0bdc012743d_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:26,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/c70b1b6ba50d45adae7874c50d5640cf, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:26,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/c70b1b6ba50d45adae7874c50d5640cf is 175, key is test_row_0/A:col10/1731883345547/Put/seqid=0 2024-11-17T22:42:26,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742387_1563 (size=31105) 2024-11-17T22:42:27,233 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/6beacd2285d34fd6ba629e79f6ece809 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/6beacd2285d34fd6ba629e79f6ece809 2024-11-17T22:42:27,236 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/B of 1e5636f3ed44c24ae3f10ff764ecff86 into 6beacd2285d34fd6ba629e79f6ece809(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:27,236 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:27,236 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/B, priority=13, startTime=1731883346807; duration=0sec 2024-11-17T22:42:27,236 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:27,236 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:B 2024-11-17T22:42:27,330 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/6f016f5aa6c04a30825f815f2e791ff2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6f016f5aa6c04a30825f815f2e791ff2 2024-11-17T22:42:27,334 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/C of 1e5636f3ed44c24ae3f10ff764ecff86 into 6f016f5aa6c04a30825f815f2e791ff2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:27,334 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:27,334 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/C, priority=13, startTime=1731883346807; duration=0sec 2024-11-17T22:42:27,334 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:27,334 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:C 2024-11-17T22:42:27,338 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/c70b1b6ba50d45adae7874c50d5640cf 2024-11-17T22:42:27,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/cace34a63d7746eba47f67a393070237 is 50, key is test_row_0/B:col10/1731883345547/Put/seqid=0 2024-11-17T22:42:27,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742388_1564 (size=12151) 2024-11-17T22:42:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:27,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:27,749 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/cace34a63d7746eba47f67a393070237 2024-11-17T22:42:27,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883407747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:27,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883407747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:27,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883407747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:27,750 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in 1e5636f3ed44c24ae3f10ff764ecff86 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:42:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:27,750 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in 1e5636f3ed44c24ae3f10ff764ecff86 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:42:27,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883407747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:27,750 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in 1e5636f3ed44c24ae3f10ff764ecff86 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:42:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:27,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883407750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:27,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:27,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/6bcb72555617461b8d76bcd72b8b1868 is 50, key is test_row_0/C:col10/1731883345547/Put/seqid=0 2024-11-17T22:42:27,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742389_1565 (size=12151) 2024-11-17T22:42:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T22:42:27,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883407857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:27,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883407857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883408067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883408067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:28,166 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/6bcb72555617461b8d76bcd72b8b1868 2024-11-17T22:42:28,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/c70b1b6ba50d45adae7874c50d5640cf as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c70b1b6ba50d45adae7874c50d5640cf 2024-11-17T22:42:28,175 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c70b1b6ba50d45adae7874c50d5640cf, entries=150, sequenceid=246, filesize=30.4 K 2024-11-17T22:42:28,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/cace34a63d7746eba47f67a393070237 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cace34a63d7746eba47f67a393070237 2024-11-17T22:42:28,182 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cace34a63d7746eba47f67a393070237, entries=150, sequenceid=246, filesize=11.9 K 2024-11-17T22:42:28,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/6bcb72555617461b8d76bcd72b8b1868 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6bcb72555617461b8d76bcd72b8b1868 2024-11-17T22:42:28,188 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6bcb72555617461b8d76bcd72b8b1868, entries=150, sequenceid=246, filesize=11.9 K 2024-11-17T22:42:28,189 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1e5636f3ed44c24ae3f10ff764ecff86 in 1307ms, sequenceid=246, compaction requested=false 2024-11-17T22:42:28,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:28,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:28,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-17T22:42:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-17T22:42:28,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-17T22:42:28,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5310 sec 2024-11-17T22:42:28,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.5360 sec 2024-11-17T22:42:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:28,378 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:28,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:42:28,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117eb1d0a321f17462985c13eb308c05118_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883347748/Put/seqid=0 2024-11-17T22:42:28,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742390_1566 (size=14944) 2024-11-17T22:42:28,390 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:28,394 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117eb1d0a321f17462985c13eb308c05118_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eb1d0a321f17462985c13eb308c05118_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:28,395 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/c10e1d9d5ed248d2bfcbab0f5af16d66, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:28,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/c10e1d9d5ed248d2bfcbab0f5af16d66 is 175, key is test_row_0/A:col10/1731883347748/Put/seqid=0 2024-11-17T22:42:28,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742391_1567 (size=39899) 2024-11-17T22:42:28,402 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/c10e1d9d5ed248d2bfcbab0f5af16d66 2024-11-17T22:42:28,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/c327bb7a320b4826a156088728e4c852 is 50, key is test_row_0/B:col10/1731883347748/Put/seqid=0 2024-11-17T22:42:28,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742392_1568 (size=12301) 2024-11-17T22:42:28,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883408428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:28,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883408430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:28,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883408537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883408537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:28,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883408746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:28,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883408746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:28,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:28,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:28,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:28,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/c327bb7a320b4826a156088728e4c852 2024-11-17T22:42:28,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/b464d5bf4c184608b6f5dc868893dc03 is 50, key is test_row_0/C:col10/1731883347748/Put/seqid=0 2024-11-17T22:42:28,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742393_1569 (size=12301) 2024-11-17T22:42:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883409057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883409057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:29,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/b464d5bf4c184608b6f5dc868893dc03 2024-11-17T22:42:29,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/c10e1d9d5ed248d2bfcbab0f5af16d66 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c10e1d9d5ed248d2bfcbab0f5af16d66 2024-11-17T22:42:29,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c10e1d9d5ed248d2bfcbab0f5af16d66, entries=200, sequenceid=261, filesize=39.0 K 2024-11-17T22:42:29,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/c327bb7a320b4826a156088728e4c852 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/c327bb7a320b4826a156088728e4c852 2024-11-17T22:42:29,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/c327bb7a320b4826a156088728e4c852, entries=150, sequenceid=261, filesize=12.0 K 2024-11-17T22:42:29,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/b464d5bf4c184608b6f5dc868893dc03 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b464d5bf4c184608b6f5dc868893dc03 2024-11-17T22:42:29,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b464d5bf4c184608b6f5dc868893dc03, entries=150, sequenceid=261, filesize=12.0 K 2024-11-17T22:42:29,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1e5636f3ed44c24ae3f10ff764ecff86 in 861ms, sequenceid=261, compaction requested=true 2024-11-17T22:42:29,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:29,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:42:29,239 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:29,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:29,239 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:29,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:42:29,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:29,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:29,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:29,240 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:29,240 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:29,240 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/B is initiating minor compaction (all files) 2024-11-17T22:42:29,240 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/A is initiating minor compaction (all files) 2024-11-17T22:42:29,240 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/B in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:29,240 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/A in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:29,240 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c19c6e18a4e148a4baf2ffeeaf556769, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c70b1b6ba50d45adae7874c50d5640cf, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c10e1d9d5ed248d2bfcbab0f5af16d66] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=100.2 K 2024-11-17T22:42:29,240 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/6beacd2285d34fd6ba629e79f6ece809, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cace34a63d7746eba47f67a393070237, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/c327bb7a320b4826a156088728e4c852] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=36.3 K 2024-11-17T22:42:29,240 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:29,240 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c19c6e18a4e148a4baf2ffeeaf556769, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c70b1b6ba50d45adae7874c50d5640cf, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c10e1d9d5ed248d2bfcbab0f5af16d66] 2024-11-17T22:42:29,241 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 6beacd2285d34fd6ba629e79f6ece809, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731883345493 2024-11-17T22:42:29,241 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting c19c6e18a4e148a4baf2ffeeaf556769, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731883345493 2024-11-17T22:42:29,241 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting cace34a63d7746eba47f67a393070237, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731883345547 2024-11-17T22:42:29,242 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting c70b1b6ba50d45adae7874c50d5640cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731883345547 2024-11-17T22:42:29,242 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting c327bb7a320b4826a156088728e4c852, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731883347748 2024-11-17T22:42:29,242 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting c10e1d9d5ed248d2bfcbab0f5af16d66, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731883347744 2024-11-17T22:42:29,259 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#B#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:29,259 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/0e63ede33e1b4a669b233c451d9c2f97 is 50, key is test_row_0/B:col10/1731883347748/Put/seqid=0 2024-11-17T22:42:29,270 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:29,285 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117ef7de74f95084f81ae28a71162061972_1e5636f3ed44c24ae3f10ff764ecff86 store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:29,287 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117ef7de74f95084f81ae28a71162061972_1e5636f3ed44c24ae3f10ff764ecff86, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:29,287 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ef7de74f95084f81ae28a71162061972_1e5636f3ed44c24ae3f10ff764ecff86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:29,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742394_1570 (size=12949) 2024-11-17T22:42:29,315 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/0e63ede33e1b4a669b233c451d9c2f97 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/0e63ede33e1b4a669b233c451d9c2f97 2024-11-17T22:42:29,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742395_1571 (size=4469) 2024-11-17T22:42:29,321 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/B of 1e5636f3ed44c24ae3f10ff764ecff86 into 0e63ede33e1b4a669b233c451d9c2f97(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:29,321 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:29,321 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/B, priority=13, startTime=1731883349239; duration=0sec 2024-11-17T22:42:29,321 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#A#compaction#482 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:29,322 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:29,322 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:B 2024-11-17T22:42:29,322 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T22:42:29,324 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/43e165337a5648c28a43be7a9dd35527 is 175, key is test_row_0/A:col10/1731883347748/Put/seqid=0 2024-11-17T22:42:29,325 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T22:42:29,326 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/C is initiating minor compaction (all files) 2024-11-17T22:42:29,326 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/C in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:29,326 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6f016f5aa6c04a30825f815f2e791ff2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6bcb72555617461b8d76bcd72b8b1868, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b464d5bf4c184608b6f5dc868893dc03] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=36.2 K 2024-11-17T22:42:29,326 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f016f5aa6c04a30825f815f2e791ff2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731883345493 2024-11-17T22:42:29,326 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bcb72555617461b8d76bcd72b8b1868, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731883345547 2024-11-17T22:42:29,327 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b464d5bf4c184608b6f5dc868893dc03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731883347748 2024-11-17T22:42:29,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742396_1572 (size=31903) 2024-11-17T22:42:29,334 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#C#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:29,334 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/aabf46ac491d47f5a499c0fd4f262aa0 is 50, key is test_row_0/C:col10/1731883347748/Put/seqid=0 2024-11-17T22:42:29,338 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/43e165337a5648c28a43be7a9dd35527 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/43e165337a5648c28a43be7a9dd35527 2024-11-17T22:42:29,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742397_1573 (size=12915) 2024-11-17T22:42:29,342 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/A of 1e5636f3ed44c24ae3f10ff764ecff86 into 43e165337a5648c28a43be7a9dd35527(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:29,342 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:29,342 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/A, priority=13, startTime=1731883349239; duration=0sec 2024-11-17T22:42:29,342 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:29,342 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:A 2024-11-17T22:42:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:29,569 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:29,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T22:42:29,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172f3293dccf5546139c29a7b78f6cefa9_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883348427/Put/seqid=0 2024-11-17T22:42:29,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742398_1574 (size=12454) 2024-11-17T22:42:29,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883409591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:29,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883409591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:29,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883409707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:29,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883409707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:29,741 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/aabf46ac491d47f5a499c0fd4f262aa0 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/aabf46ac491d47f5a499c0fd4f262aa0 2024-11-17T22:42:29,745 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/C of 1e5636f3ed44c24ae3f10ff764ecff86 into aabf46ac491d47f5a499c0fd4f262aa0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:29,745 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:29,745 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/C, priority=13, startTime=1731883349239; duration=0sec 2024-11-17T22:42:29,745 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:29,745 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:C 2024-11-17T22:42:29,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56010 deadline: 1731883409767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:29,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56070 deadline: 1731883409767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:29,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56034 deadline: 1731883409767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:29,770 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in 1e5636f3ed44c24ae3f10ff764ecff86 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:42:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:29,771 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_2' in 1e5636f3ed44c24ae3f10ff764ecff86 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:42:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:29,772 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 1a6e40b21a48,43011,1731883188895 for 'test_row_1' in 1e5636f3ed44c24ae3f10ff764ecff86 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T22:42:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T22:42:29,827 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:29,827 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:42:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-17T22:42:29,831 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:42:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T22:42:29,831 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:42:29,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:42:29,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883409917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883409917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T22:42:29,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T22:42:29,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:29,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:29,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:29,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:29,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:29,990 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:29,993 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172f3293dccf5546139c29a7b78f6cefa9_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172f3293dccf5546139c29a7b78f6cefa9_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:29,993 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/528b253063ab4ae49315d8495390aeba, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:29,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/528b253063ab4ae49315d8495390aeba is 175, key is test_row_0/A:col10/1731883348427/Put/seqid=0 2024-11-17T22:42:29,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742399_1575 (size=31255) 2024-11-17T22:42:30,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T22:42:30,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:30,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:30,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:30,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T22:42:30,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883410227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:30,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883410227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:30,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T22:42:30,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:30,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:30,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:30,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,397 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/528b253063ab4ae49315d8495390aeba 2024-11-17T22:42:30,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/62d86c8ea0114946b564107b27ae6cdc is 50, key is test_row_0/B:col10/1731883348427/Put/seqid=0 2024-11-17T22:42:30,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742400_1576 (size=12301) 2024-11-17T22:42:30,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T22:42:30,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:30,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:30,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:30,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T22:42:30,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T22:42:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:30,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883410736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:30,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883410737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:30,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T22:42:30,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:30,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:30,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:30,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:30,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/62d86c8ea0114946b564107b27ae6cdc 2024-11-17T22:42:30,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/052091b437de4032baa765720005cd2a is 50, key is test_row_0/C:col10/1731883348427/Put/seqid=0 2024-11-17T22:42:30,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742401_1577 (size=12301) 2024-11-17T22:42:30,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/052091b437de4032baa765720005cd2a 2024-11-17T22:42:30,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/528b253063ab4ae49315d8495390aeba as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/528b253063ab4ae49315d8495390aeba 2024-11-17T22:42:30,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/528b253063ab4ae49315d8495390aeba, entries=150, sequenceid=288, filesize=30.5 K 2024-11-17T22:42:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/62d86c8ea0114946b564107b27ae6cdc as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/62d86c8ea0114946b564107b27ae6cdc 2024-11-17T22:42:30,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/62d86c8ea0114946b564107b27ae6cdc, entries=150, sequenceid=288, filesize=12.0 K 2024-11-17T22:42:30,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/052091b437de4032baa765720005cd2a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/052091b437de4032baa765720005cd2a 2024-11-17T22:42:30,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/052091b437de4032baa765720005cd2a, entries=150, sequenceid=288, filesize=12.0 K 2024-11-17T22:42:30,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 1e5636f3ed44c24ae3f10ff764ecff86 in 1266ms, sequenceid=288, compaction requested=false 2024-11-17T22:42:30,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:30,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T22:42:30,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:30,898 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T22:42:30,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111784a02e4c1b1444daaa3c1676c2296367_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883349579/Put/seqid=0 2024-11-17T22:42:30,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742402_1578 (size=12454) 2024-11-17T22:42:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T22:42:31,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,317 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111784a02e4c1b1444daaa3c1676c2296367_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111784a02e4c1b1444daaa3c1676c2296367_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:31,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/4f364904e3aa4673a73200eecb0fc0ad, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:31,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/4f364904e3aa4673a73200eecb0fc0ad is 175, key is test_row_0/A:col10/1731883349579/Put/seqid=0 2024-11-17T22:42:31,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742403_1579 (size=31255) 2024-11-17T22:42:31,323 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/4f364904e3aa4673a73200eecb0fc0ad 2024-11-17T22:42:31,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/e71a0d010b8448f0a50ab8f4a19aa435 is 50, key is test_row_0/B:col10/1731883349579/Put/seqid=0 2024-11-17T22:42:31,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742404_1580 (size=12301) 2024-11-17T22:42:31,355 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/e71a0d010b8448f0a50ab8f4a19aa435 2024-11-17T22:42:31,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/63ae3006abe64162a6e9c92bcb6a57db is 50, key is test_row_0/C:col10/1731883349579/Put/seqid=0 2024-11-17T22:42:31,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742405_1581 (size=12301) 2024-11-17T22:42:31,384 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/63ae3006abe64162a6e9c92bcb6a57db 2024-11-17T22:42:31,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/4f364904e3aa4673a73200eecb0fc0ad as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4f364904e3aa4673a73200eecb0fc0ad 2024-11-17T22:42:31,395 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4f364904e3aa4673a73200eecb0fc0ad, entries=150, sequenceid=301, filesize=30.5 K 2024-11-17T22:42:31,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/e71a0d010b8448f0a50ab8f4a19aa435 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/e71a0d010b8448f0a50ab8f4a19aa435 2024-11-17T22:42:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,399 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/e71a0d010b8448f0a50ab8f4a19aa435, entries=150, sequenceid=301, filesize=12.0 K 2024-11-17T22:42:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/63ae3006abe64162a6e9c92bcb6a57db as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/63ae3006abe64162a6e9c92bcb6a57db 2024-11-17T22:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,415 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/63ae3006abe64162a6e9c92bcb6a57db, entries=150, sequenceid=301, filesize=12.0 K 2024-11-17T22:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,416 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 1e5636f3ed44c24ae3f10ff764ecff86 in 518ms, sequenceid=301, compaction requested=true 2024-11-17T22:42:31,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:31,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:31,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-17T22:42:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-17T22:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-17T22:42:31,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5860 sec 2024-11-17T22:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.5930 sec 2024-11-17T22:42:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:31,834 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T22:42:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T22:42:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:31,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170ae3b65cfdd74f9594f77636a53dec29_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883351819/Put/seqid=0 2024-11-17T22:42:31,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742406_1582 (size=20074) 2024-11-17T22:42:31,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883411911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:31,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883411914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T22:42:31,977 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:31,977 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T22:42:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-17T22:42:31,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T22:42:31,979 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T22:42:31,979 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T22:42:31,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T22:42:32,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883412027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:32,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883412027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T22:42:32,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T22:42:32,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:32,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883412237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883412237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:32,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T22:42:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,291 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:32,296 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170ae3b65cfdd74f9594f77636a53dec29_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170ae3b65cfdd74f9594f77636a53dec29_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T22:42:32,297 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/f0e57566e63448ccb25e268f9bc46688, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:32,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/f0e57566e63448ccb25e268f9bc46688 is 175, key is test_row_0/A:col10/1731883351819/Put/seqid=0 2024-11-17T22:42:32,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742407_1583 (size=57333) 2024-11-17T22:42:32,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T22:42:32,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:32,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883412547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883412547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:32,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T22:42:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T22:42:32,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/f0e57566e63448ccb25e268f9bc46688 2024-11-17T22:42:32,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/b682118238784f4085294e8354a66e17 is 50, key is test_row_0/B:col10/1731883351819/Put/seqid=0 2024-11-17T22:42:32,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742408_1584 (size=12301) 2024-11-17T22:42:32,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T22:42:32,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:32,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T22:42:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:32,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T22:42:33,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:33,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56062 deadline: 1731883413057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:33,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T22:42:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43011 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55976 deadline: 1731883413057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 , the old value is region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T22:42:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., hostname=1a6e40b21a48,43011,1731883188895, seqNum=5 because the exception is null or not the one we care about 2024-11-17T22:42:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T22:42:33,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/b682118238784f4085294e8354a66e17 2024-11-17T22:42:33,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/9d3409eb5e4a4ec7add9376653e36a87 is 50, key is test_row_0/C:col10/1731883351819/Put/seqid=0 2024-11-17T22:42:33,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742409_1585 (size=12301) 2024-11-17T22:42:33,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T22:42:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T22:42:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,406 INFO [Thread-2223 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2223. 2024-11-17T22:42:33,406 DEBUG [Thread-2223 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:33,406 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,406 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:33,407 INFO [Thread-2225 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2225. 2024-11-17T22:42:33,407 DEBUG [Thread-2225 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:33,407 DEBUG [Thread-2225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,407 DEBUG [Thread-2225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:33,409 INFO [Thread-2231 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2231. 2024-11-17T22:42:33,409 DEBUG [Thread-2231 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:33,409 DEBUG [Thread-2231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,409 DEBUG [Thread-2231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:33,410 INFO [Thread-2227 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2227. 2024-11-17T22:42:33,410 DEBUG [Thread-2227 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:33,410 DEBUG [Thread-2227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,410 DEBUG [Thread-2227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:33,411 INFO [Thread-2229 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2229. 2024-11-17T22:42:33,411 DEBUG [Thread-2229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:33,411 DEBUG [Thread-2229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,411 DEBUG [Thread-2229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:33,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T22:42:33,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:33,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T22:42:33,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/9d3409eb5e4a4ec7add9376653e36a87 2024-11-17T22:42:33,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/f0e57566e63448ccb25e268f9bc46688 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f0e57566e63448ccb25e268f9bc46688 2024-11-17T22:42:33,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f0e57566e63448ccb25e268f9bc46688, entries=300, sequenceid=312, filesize=56.0 K 2024-11-17T22:42:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/b682118238784f4085294e8354a66e17 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b682118238784f4085294e8354a66e17 2024-11-17T22:42:33,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b682118238784f4085294e8354a66e17, entries=150, sequenceid=312, filesize=12.0 K 2024-11-17T22:42:33,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/9d3409eb5e4a4ec7add9376653e36a87 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9d3409eb5e4a4ec7add9376653e36a87 2024-11-17T22:42:33,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9d3409eb5e4a4ec7add9376653e36a87, entries=150, sequenceid=312, filesize=12.0 K 2024-11-17T22:42:33,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1e5636f3ed44c24ae3f10ff764ecff86 in 1722ms, sequenceid=312, compaction requested=true 2024-11-17T22:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T22:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T22:42:33,556 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e5636f3ed44c24ae3f10ff764ecff86:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T22:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:33,556 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:42:33,557 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151746 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:42:33,557 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:42:33,557 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/A is initiating minor compaction (all files) 2024-11-17T22:42:33,557 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/B is initiating minor compaction (all files) 2024-11-17T22:42:33,557 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/A in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,557 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/B in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,557 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/0e63ede33e1b4a669b233c451d9c2f97, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/62d86c8ea0114946b564107b27ae6cdc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/e71a0d010b8448f0a50ab8f4a19aa435, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b682118238784f4085294e8354a66e17] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=48.7 K 2024-11-17T22:42:33,557 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/43e165337a5648c28a43be7a9dd35527, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/528b253063ab4ae49315d8495390aeba, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4f364904e3aa4673a73200eecb0fc0ad, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f0e57566e63448ccb25e268f9bc46688] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=148.2 K 2024-11-17T22:42:33,557 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,557 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. files: [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/43e165337a5648c28a43be7a9dd35527, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/528b253063ab4ae49315d8495390aeba, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4f364904e3aa4673a73200eecb0fc0ad, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f0e57566e63448ccb25e268f9bc46688] 2024-11-17T22:42:33,557 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e63ede33e1b4a669b233c451d9c2f97, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731883347748 2024-11-17T22:42:33,557 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e165337a5648c28a43be7a9dd35527, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731883347748 2024-11-17T22:42:33,558 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 62d86c8ea0114946b564107b27ae6cdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731883348421 2024-11-17T22:42:33,558 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 528b253063ab4ae49315d8495390aeba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731883348421 2024-11-17T22:42:33,558 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting e71a0d010b8448f0a50ab8f4a19aa435, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731883349579 2024-11-17T22:42:33,558 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f364904e3aa4673a73200eecb0fc0ad, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731883349579 2024-11-17T22:42:33,558 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting b682118238784f4085294e8354a66e17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731883351809 2024-11-17T22:42:33,558 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0e57566e63448ccb25e268f9bc46688, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731883351800 2024-11-17T22:42:33,562 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:33,563 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#B#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:33,563 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117f47944a907a94def8a40b1cf9443cd2f_1e5636f3ed44c24ae3f10ff764ecff86 store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:33,564 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/20fadcaf6b854bc2b6239fc66c68f9ba is 50, key is test_row_0/B:col10/1731883351819/Put/seqid=0 2024-11-17T22:42:33,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742410_1586 (size=13085) 2024-11-17T22:42:33,566 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117f47944a907a94def8a40b1cf9443cd2f_1e5636f3ed44c24ae3f10ff764ecff86, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:33,567 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f47944a907a94def8a40b1cf9443cd2f_1e5636f3ed44c24ae3f10ff764ecff86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:33,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742411_1587 (size=4469) 2024-11-17T22:42:33,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T22:42:33,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,660 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T22:42:33,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117394cac633dab4655838cd6a828d9f515_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883351893/Put/seqid=0 2024-11-17T22:42:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742412_1588 (size=12454) 2024-11-17T22:42:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43011 {}] regionserver.HRegion(8855): Flush requested on 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:33,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. as already flushing 2024-11-17T22:42:33,779 INFO [Thread-2216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2216. 2024-11-17T22:42:33,780 DEBUG [Thread-2216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:33,780 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,780 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:33,789 INFO [Thread-2220 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2220. 2024-11-17T22:42:33,789 DEBUG [Thread-2220 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:33,789 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,789 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:33,800 INFO [Thread-2218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2218. 2024-11-17T22:42:33,800 DEBUG [Thread-2218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:33,800 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,800 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:33,800 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:33,972 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#A#compaction#494 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:33,973 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/83a00463ab714b7b97daf924bc3d26f1 is 175, key is test_row_0/A:col10/1731883351819/Put/seqid=0 2024-11-17T22:42:33,978 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/20fadcaf6b854bc2b6239fc66c68f9ba as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/20fadcaf6b854bc2b6239fc66c68f9ba 2024-11-17T22:42:33,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742413_1589 (size=32039) 2024-11-17T22:42:33,983 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/B of 1e5636f3ed44c24ae3f10ff764ecff86 into 20fadcaf6b854bc2b6239fc66c68f9ba(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:33,983 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:33,983 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/B, priority=12, startTime=1731883353556; duration=0sec 2024-11-17T22:42:33,984 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T22:42:33,984 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:B 2024-11-17T22:42:33,984 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T22:42:33,985 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T22:42:33,985 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1541): 1e5636f3ed44c24ae3f10ff764ecff86/C is initiating minor compaction (all files) 2024-11-17T22:42:33,985 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e5636f3ed44c24ae3f10ff764ecff86/C in TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:33,985 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/aabf46ac491d47f5a499c0fd4f262aa0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/052091b437de4032baa765720005cd2a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/63ae3006abe64162a6e9c92bcb6a57db, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9d3409eb5e4a4ec7add9376653e36a87] into tmpdir=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp, totalSize=48.7 K 2024-11-17T22:42:33,985 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting aabf46ac491d47f5a499c0fd4f262aa0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731883347748 2024-11-17T22:42:33,986 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 052091b437de4032baa765720005cd2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731883348421 2024-11-17T22:42:33,986 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 63ae3006abe64162a6e9c92bcb6a57db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731883349579 2024-11-17T22:42:33,986 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d3409eb5e4a4ec7add9376653e36a87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731883351809 2024-11-17T22:42:33,994 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e5636f3ed44c24ae3f10ff764ecff86#C#compaction#496 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T22:42:33,994 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/d2fe44918585423fb7c9d47fa6bab75b is 50, key is test_row_0/C:col10/1731883351819/Put/seqid=0 2024-11-17T22:42:33,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742414_1590 (size=13051) 2024-11-17T22:42:34,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:34,079 INFO [Thread-2214 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2214. 2024-11-17T22:42:34,080 DEBUG [Thread-2214 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:34,080 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:34,080 INFO [Thread-2212 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2212. 2024-11-17T22:42:34,080 DEBUG [Thread-2212 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T22:42:34,080 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:34,080 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:34,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:34,080 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:34,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:34,084 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117394cac633dab4655838cd6a828d9f515_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117394cac633dab4655838cd6a828d9f515_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:34,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/63bde1294b6f4a0db3d80b109dd6d60b, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:34,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/63bde1294b6f4a0db3d80b109dd6d60b is 175, key is test_row_0/A:col10/1731883351893/Put/seqid=0 2024-11-17T22:42:34,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742415_1591 (size=31255) 2024-11-17T22:42:34,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T22:42:34,383 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/83a00463ab714b7b97daf924bc3d26f1 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/83a00463ab714b7b97daf924bc3d26f1 2024-11-17T22:42:34,386 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/A of 1e5636f3ed44c24ae3f10ff764ecff86 into 83a00463ab714b7b97daf924bc3d26f1(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:34,386 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:34,386 INFO [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/A, priority=12, startTime=1731883353556; duration=0sec 2024-11-17T22:42:34,386 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:34,386 DEBUG [RS:0;1a6e40b21a48:43011-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:A 2024-11-17T22:42:34,402 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/d2fe44918585423fb7c9d47fa6bab75b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/d2fe44918585423fb7c9d47fa6bab75b 2024-11-17T22:42:34,406 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e5636f3ed44c24ae3f10ff764ecff86/C of 1e5636f3ed44c24ae3f10ff764ecff86 into d2fe44918585423fb7c9d47fa6bab75b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T22:42:34,406 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:34,406 INFO [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., storeName=1e5636f3ed44c24ae3f10ff764ecff86/C, priority=12, startTime=1731883353556; duration=0sec 2024-11-17T22:42:34,406 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T22:42:34,406 DEBUG [RS:0;1a6e40b21a48:43011-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e5636f3ed44c24ae3f10ff764ecff86:C 2024-11-17T22:42:34,492 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/63bde1294b6f4a0db3d80b109dd6d60b 2024-11-17T22:42:34,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/e0b5a3e8ec8f4a4e879b8b3a6fdd8522 is 50, key is test_row_0/B:col10/1731883351893/Put/seqid=0 2024-11-17T22:42:34,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742416_1592 (size=12301) 2024-11-17T22:42:34,901 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/e0b5a3e8ec8f4a4e879b8b3a6fdd8522 2024-11-17T22:42:34,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/88ad51f3af6a43739c0ec12549dbf83e is 50, key is test_row_0/C:col10/1731883351893/Put/seqid=0 2024-11-17T22:42:34,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742417_1593 (size=12301) 2024-11-17T22:42:35,311 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/88ad51f3af6a43739c0ec12549dbf83e 2024-11-17T22:42:35,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/63bde1294b6f4a0db3d80b109dd6d60b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/63bde1294b6f4a0db3d80b109dd6d60b 2024-11-17T22:42:35,318 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/63bde1294b6f4a0db3d80b109dd6d60b, entries=150, sequenceid=337, filesize=30.5 K 2024-11-17T22:42:35,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/e0b5a3e8ec8f4a4e879b8b3a6fdd8522 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/e0b5a3e8ec8f4a4e879b8b3a6fdd8522 2024-11-17T22:42:35,321 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/e0b5a3e8ec8f4a4e879b8b3a6fdd8522, entries=150, sequenceid=337, filesize=12.0 K 2024-11-17T22:42:35,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/88ad51f3af6a43739c0ec12549dbf83e as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/88ad51f3af6a43739c0ec12549dbf83e 2024-11-17T22:42:35,324 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/88ad51f3af6a43739c0ec12549dbf83e, entries=150, sequenceid=337, filesize=12.0 K 2024-11-17T22:42:35,324 INFO [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for 1e5636f3ed44c24ae3f10ff764ecff86 in 1664ms, sequenceid=337, compaction requested=false 2024-11-17T22:42:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 1e5636f3ed44c24ae3f10ff764ecff86: 2024-11-17T22:42:35,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:35,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a6e40b21a48:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-17T22:42:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-17T22:42:35,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-17T22:42:35,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3470 sec 2024-11-17T22:42:35,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 3.3490 sec 2024-11-17T22:42:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T22:42:36,137 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:36,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T22:42:36,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-17T22:42:36,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-17T22:42:36,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-17T22:42:36,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-17T22:42:36,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-17T22:42:36,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T22:42:36,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3248 2024-11-17T22:42:36,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3327 2024-11-17T22:42:36,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3321 2024-11-17T22:42:36,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3248 2024-11-17T22:42:36,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3334 2024-11-17T22:42:36,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T22:42:36,138 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T22:42:36,138 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T22:42:36,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:36,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:36,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:36,139 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T22:42:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T22:42:36,141 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883356141"}]},"ts":"1731883356141"} 2024-11-17T22:42:36,142 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T22:42:36,142 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T22:42:36,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T22:42:36,143 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, UNASSIGN}] 2024-11-17T22:42:36,144 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, UNASSIGN 2024-11-17T22:42:36,144 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=1e5636f3ed44c24ae3f10ff764ecff86, regionState=CLOSING, regionLocation=1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:36,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, UNASSIGN because future has completed 2024-11-17T22:42:36,145 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T22:42:36,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895}] 2024-11-17T22:42:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T22:42:36,298 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:36,298 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T22:42:36,298 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing 1e5636f3ed44c24ae3f10ff764ecff86, disabling compactions & flushes 2024-11-17T22:42:36,298 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:36,299 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:36,299 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. after waiting 0 ms 2024-11-17T22:42:36,299 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:36,299 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing 1e5636f3ed44c24ae3f10ff764ecff86 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T22:42:36,312 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e4ec0904854e4752bf7fd77108f25472_1e5636f3ed44c24ae3f10ff764ecff86 is 50, key is test_row_0/A:col10/1731883354077/Put/seqid=0 2024-11-17T22:42:36,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742418_1594 (size=12454) 2024-11-17T22:42:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T22:42:36,717 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T22:42:36,726 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e4ec0904854e4752bf7fd77108f25472_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e4ec0904854e4752bf7fd77108f25472_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:36,728 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/7b31a5e166b549eaa9131203a1fbd98b, store: [table=TestAcidGuarantees family=A region=1e5636f3ed44c24ae3f10ff764ecff86] 2024-11-17T22:42:36,729 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/7b31a5e166b549eaa9131203a1fbd98b is 175, key is test_row_0/A:col10/1731883354077/Put/seqid=0 2024-11-17T22:42:36,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742419_1595 (size=31255) 2024-11-17T22:42:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T22:42:37,135 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/7b31a5e166b549eaa9131203a1fbd98b 2024-11-17T22:42:37,148 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/fb25a5b79b204a318474df26adccee6a is 50, key is test_row_0/B:col10/1731883354077/Put/seqid=0 2024-11-17T22:42:37,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742420_1596 (size=12301) 2024-11-17T22:42:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T22:42:37,552 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/fb25a5b79b204a318474df26adccee6a 2024-11-17T22:42:37,557 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/0d8def3bbfd542aabe82fca2fd4736c6 is 50, key is test_row_0/C:col10/1731883354077/Put/seqid=0 2024-11-17T22:42:37,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742421_1597 (size=12301) 2024-11-17T22:42:37,961 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/0d8def3bbfd542aabe82fca2fd4736c6 2024-11-17T22:42:37,964 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/A/7b31a5e166b549eaa9131203a1fbd98b as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/7b31a5e166b549eaa9131203a1fbd98b 2024-11-17T22:42:37,967 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/7b31a5e166b549eaa9131203a1fbd98b, entries=150, sequenceid=348, filesize=30.5 K 2024-11-17T22:42:37,968 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/B/fb25a5b79b204a318474df26adccee6a as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/fb25a5b79b204a318474df26adccee6a 2024-11-17T22:42:37,971 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/fb25a5b79b204a318474df26adccee6a, entries=150, sequenceid=348, filesize=12.0 K 2024-11-17T22:42:37,972 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/.tmp/C/0d8def3bbfd542aabe82fca2fd4736c6 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/0d8def3bbfd542aabe82fca2fd4736c6 2024-11-17T22:42:37,974 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/0d8def3bbfd542aabe82fca2fd4736c6, entries=150, sequenceid=348, filesize=12.0 K 2024-11-17T22:42:37,975 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1e5636f3ed44c24ae3f10ff764ecff86 in 1676ms, sequenceid=348, compaction requested=true 2024-11-17T22:42:37,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/7b60c22263554005b6650cb1c5ed756e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/bd761915fef04406914b93eac024b966, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4ef80568c8154245869d1df20ab67438, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ac19d11a223a4c5cbf0f939123bb8d3b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5c170c1966b942bf9249d8be0498cca5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/37ebc698a09b420494b15326e114a9ed, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/2f4cff1198624983aff42e37fd614ffe, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5742c6e09dde44908b9d37bc8edc342e, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/8d96c693ce2e450cb81b6c8667f429fd, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/63fca6d123bd4748ab12fa732238f393, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ab8eebde61dc42ca864a1ff25e037c72, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/faf3923d4df44fe096e3f5b39f5af05b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f8edd65f7bee43a1a81054e0a31ac0fc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5cbdc28c0e274c119426a5c8c45aa93a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/fcf96ad2280641bb952945342a3caa31, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/926671b625ed49d49b7629e4d2d67fe7, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c19c6e18a4e148a4baf2ffeeaf556769, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c70b1b6ba50d45adae7874c50d5640cf, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c10e1d9d5ed248d2bfcbab0f5af16d66, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/43e165337a5648c28a43be7a9dd35527, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/528b253063ab4ae49315d8495390aeba, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4f364904e3aa4673a73200eecb0fc0ad, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f0e57566e63448ccb25e268f9bc46688] to archive 2024-11-17T22:42:37,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:42:37,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/7b60c22263554005b6650cb1c5ed756e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/7b60c22263554005b6650cb1c5ed756e 2024-11-17T22:42:37,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/bd761915fef04406914b93eac024b966 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/bd761915fef04406914b93eac024b966 2024-11-17T22:42:37,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4ef80568c8154245869d1df20ab67438 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4ef80568c8154245869d1df20ab67438 2024-11-17T22:42:37,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ac19d11a223a4c5cbf0f939123bb8d3b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ac19d11a223a4c5cbf0f939123bb8d3b 2024-11-17T22:42:37,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5c170c1966b942bf9249d8be0498cca5 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5c170c1966b942bf9249d8be0498cca5 2024-11-17T22:42:37,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/37ebc698a09b420494b15326e114a9ed to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/37ebc698a09b420494b15326e114a9ed 2024-11-17T22:42:37,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/2f4cff1198624983aff42e37fd614ffe to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/2f4cff1198624983aff42e37fd614ffe 2024-11-17T22:42:37,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5742c6e09dde44908b9d37bc8edc342e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5742c6e09dde44908b9d37bc8edc342e 2024-11-17T22:42:37,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/8d96c693ce2e450cb81b6c8667f429fd to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/8d96c693ce2e450cb81b6c8667f429fd 2024-11-17T22:42:37,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/63fca6d123bd4748ab12fa732238f393 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/63fca6d123bd4748ab12fa732238f393 2024-11-17T22:42:37,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ab8eebde61dc42ca864a1ff25e037c72 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/ab8eebde61dc42ca864a1ff25e037c72 2024-11-17T22:42:37,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/faf3923d4df44fe096e3f5b39f5af05b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/faf3923d4df44fe096e3f5b39f5af05b 2024-11-17T22:42:37,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f8edd65f7bee43a1a81054e0a31ac0fc to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f8edd65f7bee43a1a81054e0a31ac0fc 2024-11-17T22:42:37,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5cbdc28c0e274c119426a5c8c45aa93a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/5cbdc28c0e274c119426a5c8c45aa93a 2024-11-17T22:42:37,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/fcf96ad2280641bb952945342a3caa31 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/fcf96ad2280641bb952945342a3caa31 2024-11-17T22:42:37,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/926671b625ed49d49b7629e4d2d67fe7 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/926671b625ed49d49b7629e4d2d67fe7 2024-11-17T22:42:37,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c19c6e18a4e148a4baf2ffeeaf556769 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c19c6e18a4e148a4baf2ffeeaf556769 2024-11-17T22:42:37,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c70b1b6ba50d45adae7874c50d5640cf to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c70b1b6ba50d45adae7874c50d5640cf 2024-11-17T22:42:37,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c10e1d9d5ed248d2bfcbab0f5af16d66 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/c10e1d9d5ed248d2bfcbab0f5af16d66 2024-11-17T22:42:37,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/43e165337a5648c28a43be7a9dd35527 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/43e165337a5648c28a43be7a9dd35527 2024-11-17T22:42:37,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/528b253063ab4ae49315d8495390aeba to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/528b253063ab4ae49315d8495390aeba 2024-11-17T22:42:37,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4f364904e3aa4673a73200eecb0fc0ad to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/4f364904e3aa4673a73200eecb0fc0ad 2024-11-17T22:42:37,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f0e57566e63448ccb25e268f9bc46688 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/f0e57566e63448ccb25e268f9bc46688 2024-11-17T22:42:37,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cd7476ef8998443c834d71e112b2b8c2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/faff33ae15054d819e657bea6fa65a27, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/d7a07b1ee8b84710b1244c2fd36d09f0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/2935120329bc41ce8418c67acf671fbe, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5f5075a8ba684801bd92d2b971f594aa, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/02380a74d4a84677851ee603f49182ea, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5b7eec51f75743e5b3c535d19fab2895, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/11cd8dc344c44d2b860057903165284b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/9df39d3682af42fa8c8a9ef35045e1c8, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5ef8f4b3fcfb49c19b76b96d230dd2fa, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/d57697627ab64018b2cc49471baa9714, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5e2ba17661d14094bec00c33a78baa80, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/ddd1316d4ae44574923a9db884c8df37, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b75d4c615d0c4ecc926aa60d73a20185, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/89970e6c93184de6bd16907ff3be8113, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/6beacd2285d34fd6ba629e79f6ece809, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/0f28738c7f1041ea908fe0406b6b40f1, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cace34a63d7746eba47f67a393070237, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/0e63ede33e1b4a669b233c451d9c2f97, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/c327bb7a320b4826a156088728e4c852, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/62d86c8ea0114946b564107b27ae6cdc, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/e71a0d010b8448f0a50ab8f4a19aa435, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b682118238784f4085294e8354a66e17] to archive 2024-11-17T22:42:37,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:42:37,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cd7476ef8998443c834d71e112b2b8c2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cd7476ef8998443c834d71e112b2b8c2 2024-11-17T22:42:38,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/faff33ae15054d819e657bea6fa65a27 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/faff33ae15054d819e657bea6fa65a27 2024-11-17T22:42:38,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/d7a07b1ee8b84710b1244c2fd36d09f0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/d7a07b1ee8b84710b1244c2fd36d09f0 2024-11-17T22:42:38,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/2935120329bc41ce8418c67acf671fbe to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/2935120329bc41ce8418c67acf671fbe 2024-11-17T22:42:38,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5f5075a8ba684801bd92d2b971f594aa to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5f5075a8ba684801bd92d2b971f594aa 2024-11-17T22:42:38,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/02380a74d4a84677851ee603f49182ea to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/02380a74d4a84677851ee603f49182ea 2024-11-17T22:42:38,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5b7eec51f75743e5b3c535d19fab2895 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5b7eec51f75743e5b3c535d19fab2895 2024-11-17T22:42:38,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/11cd8dc344c44d2b860057903165284b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/11cd8dc344c44d2b860057903165284b 2024-11-17T22:42:38,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/9df39d3682af42fa8c8a9ef35045e1c8 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/9df39d3682af42fa8c8a9ef35045e1c8 2024-11-17T22:42:38,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5ef8f4b3fcfb49c19b76b96d230dd2fa to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5ef8f4b3fcfb49c19b76b96d230dd2fa 2024-11-17T22:42:38,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/d57697627ab64018b2cc49471baa9714 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/d57697627ab64018b2cc49471baa9714 2024-11-17T22:42:38,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5e2ba17661d14094bec00c33a78baa80 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/5e2ba17661d14094bec00c33a78baa80 2024-11-17T22:42:38,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/ddd1316d4ae44574923a9db884c8df37 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/ddd1316d4ae44574923a9db884c8df37 2024-11-17T22:42:38,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b75d4c615d0c4ecc926aa60d73a20185 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b75d4c615d0c4ecc926aa60d73a20185 2024-11-17T22:42:38,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/89970e6c93184de6bd16907ff3be8113 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/89970e6c93184de6bd16907ff3be8113 2024-11-17T22:42:38,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/6beacd2285d34fd6ba629e79f6ece809 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/6beacd2285d34fd6ba629e79f6ece809 2024-11-17T22:42:38,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/0f28738c7f1041ea908fe0406b6b40f1 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/0f28738c7f1041ea908fe0406b6b40f1 2024-11-17T22:42:38,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cace34a63d7746eba47f67a393070237 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/cace34a63d7746eba47f67a393070237 2024-11-17T22:42:38,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/0e63ede33e1b4a669b233c451d9c2f97 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/0e63ede33e1b4a669b233c451d9c2f97 2024-11-17T22:42:38,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/c327bb7a320b4826a156088728e4c852 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/c327bb7a320b4826a156088728e4c852 2024-11-17T22:42:38,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/62d86c8ea0114946b564107b27ae6cdc to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/62d86c8ea0114946b564107b27ae6cdc 2024-11-17T22:42:38,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/e71a0d010b8448f0a50ab8f4a19aa435 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/e71a0d010b8448f0a50ab8f4a19aa435 2024-11-17T22:42:38,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b682118238784f4085294e8354a66e17 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/b682118238784f4085294e8354a66e17 2024-11-17T22:42:38,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/911d5ce95ff1401dabec9a58c76af4a4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/218f68fb9063448cb864b2124db2567f, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/d64c2ecefcb6483387416e4cfaae9da4, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/092e0c7ca5a342bca8655369141c4c68, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2eaad9b393314736b5c46481d68e7149, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/595b872e26e445bf8cede5e8656edfe5, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b1f28c31a2734e21ae067ffdcbccc51a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/ef7f95a80ec34e8eafa74a3d4bf16211, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9ce430b5c6b44b859fbbc11b793acecd, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2116da584e3541a48ac9c3119a37ac6a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7656c305d5ab4b9c9f196d23957d9de2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6a047a2702b24276a8702b6e85ba2e3c, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6478c1ff16f04b5ba0ddc0add353f71d, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9ddbc3a6c07a42d0a267c1dd5659935b, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6f016f5aa6c04a30825f815f2e791ff2, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7465dbc629f6461fa4bbab36f3dae8ad, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6bcb72555617461b8d76bcd72b8b1868, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/aabf46ac491d47f5a499c0fd4f262aa0, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b464d5bf4c184608b6f5dc868893dc03, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/052091b437de4032baa765720005cd2a, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/63ae3006abe64162a6e9c92bcb6a57db, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9d3409eb5e4a4ec7add9376653e36a87] to archive 2024-11-17T22:42:38,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T22:42:38,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/911d5ce95ff1401dabec9a58c76af4a4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/911d5ce95ff1401dabec9a58c76af4a4 2024-11-17T22:42:38,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/218f68fb9063448cb864b2124db2567f to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/218f68fb9063448cb864b2124db2567f 2024-11-17T22:42:38,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/d64c2ecefcb6483387416e4cfaae9da4 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/d64c2ecefcb6483387416e4cfaae9da4 2024-11-17T22:42:38,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/092e0c7ca5a342bca8655369141c4c68 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/092e0c7ca5a342bca8655369141c4c68 2024-11-17T22:42:38,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2eaad9b393314736b5c46481d68e7149 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2eaad9b393314736b5c46481d68e7149 2024-11-17T22:42:38,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/595b872e26e445bf8cede5e8656edfe5 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/595b872e26e445bf8cede5e8656edfe5 2024-11-17T22:42:38,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b1f28c31a2734e21ae067ffdcbccc51a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b1f28c31a2734e21ae067ffdcbccc51a 2024-11-17T22:42:38,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/ef7f95a80ec34e8eafa74a3d4bf16211 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/ef7f95a80ec34e8eafa74a3d4bf16211 2024-11-17T22:42:38,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9ce430b5c6b44b859fbbc11b793acecd to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9ce430b5c6b44b859fbbc11b793acecd 2024-11-17T22:42:38,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2116da584e3541a48ac9c3119a37ac6a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/2116da584e3541a48ac9c3119a37ac6a 2024-11-17T22:42:38,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7656c305d5ab4b9c9f196d23957d9de2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7656c305d5ab4b9c9f196d23957d9de2 2024-11-17T22:42:38,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6a047a2702b24276a8702b6e85ba2e3c to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6a047a2702b24276a8702b6e85ba2e3c 2024-11-17T22:42:38,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6478c1ff16f04b5ba0ddc0add353f71d to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6478c1ff16f04b5ba0ddc0add353f71d 2024-11-17T22:42:38,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9ddbc3a6c07a42d0a267c1dd5659935b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9ddbc3a6c07a42d0a267c1dd5659935b 2024-11-17T22:42:38,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6f016f5aa6c04a30825f815f2e791ff2 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6f016f5aa6c04a30825f815f2e791ff2 2024-11-17T22:42:38,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7465dbc629f6461fa4bbab36f3dae8ad to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/7465dbc629f6461fa4bbab36f3dae8ad 2024-11-17T22:42:38,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6bcb72555617461b8d76bcd72b8b1868 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/6bcb72555617461b8d76bcd72b8b1868 2024-11-17T22:42:38,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/aabf46ac491d47f5a499c0fd4f262aa0 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/aabf46ac491d47f5a499c0fd4f262aa0 2024-11-17T22:42:38,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b464d5bf4c184608b6f5dc868893dc03 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/b464d5bf4c184608b6f5dc868893dc03 2024-11-17T22:42:38,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/052091b437de4032baa765720005cd2a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/052091b437de4032baa765720005cd2a 2024-11-17T22:42:38,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/63ae3006abe64162a6e9c92bcb6a57db to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/63ae3006abe64162a6e9c92bcb6a57db 2024-11-17T22:42:38,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9d3409eb5e4a4ec7add9376653e36a87 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/9d3409eb5e4a4ec7add9376653e36a87 2024-11-17T22:42:38,035 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=4 2024-11-17T22:42:38,035 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. 2024-11-17T22:42:38,036 DEBUG [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for 1e5636f3ed44c24ae3f10ff764ecff86: Waiting for close lock at 1731883356298Running coprocessor pre-close hooks at 1731883356298Disabling compacts and flushes for region at 1731883356298Disabling writes for close at 1731883356299 (+1 ms)Obtaining lock to block concurrent updates at 1731883356299Preparing flush snapshotting stores in 1e5636f3ed44c24ae3f10ff764ecff86 at 1731883356299Finished memstore snapshotting TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731883356299Flushing stores of TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86. at 1731883356301 (+2 ms)Flushing 1e5636f3ed44c24ae3f10ff764ecff86/A: creating writer at 1731883356301Flushing mob file 1e5636f3ed44c24ae3f10ff764ecff86/A: appending metadata at 1731883356312 (+11 ms)Flushing mob file 1e5636f3ed44c24ae3f10ff764ecff86/A: closing flushed file at 1731883356312Flushing 1e5636f3ed44c24ae3f10ff764ecff86/A: appending metadata at 1731883356728 (+416 ms)Flushing 1e5636f3ed44c24ae3f10ff764ecff86/A: closing flushed file at 1731883356728Flushing 1e5636f3ed44c24ae3f10ff764ecff86/B: creating writer at 1731883357144 (+416 ms)Flushing 1e5636f3ed44c24ae3f10ff764ecff86/B: appending metadata at 1731883357147 (+3 ms)Flushing 1e5636f3ed44c24ae3f10ff764ecff86/B: closing flushed file at 1731883357147Flushing 1e5636f3ed44c24ae3f10ff764ecff86/C: creating writer at 1731883357555 (+408 ms)Flushing 1e5636f3ed44c24ae3f10ff764ecff86/C: appending metadata at 1731883357557 (+2 ms)Flushing 1e5636f3ed44c24ae3f10ff764ecff86/C: closing flushed file at 1731883357557Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a0683a5: reopening flushed file at 1731883357964 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a5ffb00: reopening flushed file at 1731883357967 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48fbf541: reopening flushed file at 1731883357971 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1e5636f3ed44c24ae3f10ff764ecff86 in 1676ms, sequenceid=348, compaction requested=true at 1731883357975 (+4 ms)Writing region close event to WAL at 1731883358033 (+58 ms)Running coprocessor post-close hooks at 1731883358035 (+2 ms)Closed at 1731883358035 2024-11-17T22:42:38,037 INFO [RS_CLOSE_REGION-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed 1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,037 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=1e5636f3ed44c24ae3f10ff764ecff86, regionState=CLOSED 2024-11-17T22:42:38,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 because future has completed 2024-11-17T22:42:38,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-17T22:42:38,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e5636f3ed44c24ae3f10ff764ecff86, server=1a6e40b21a48,43011,1731883188895 in 1.8940 sec 2024-11-17T22:42:38,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-17T22:42:38,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e5636f3ed44c24ae3f10ff764ecff86, UNASSIGN in 1.8970 sec 2024-11-17T22:42:38,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-17T22:42:38,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9000 sec 2024-11-17T22:42:38,045 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731883358045"}]},"ts":"1731883358045"} 2024-11-17T22:42:38,046 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T22:42:38,046 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T22:42:38,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9070 sec 2024-11-17T22:42:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T22:42:38,287 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:38,288 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T22:42:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:38,290 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-17T22:42:38,290 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:38,293 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,294 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C, FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/recovered.edits] 2024-11-17T22:42:38,296 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/63bde1294b6f4a0db3d80b109dd6d60b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/63bde1294b6f4a0db3d80b109dd6d60b 2024-11-17T22:42:38,298 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/7b31a5e166b549eaa9131203a1fbd98b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/7b31a5e166b549eaa9131203a1fbd98b 2024-11-17T22:42:38,299 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/83a00463ab714b7b97daf924bc3d26f1 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/A/83a00463ab714b7b97daf924bc3d26f1 2024-11-17T22:42:38,300 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/20fadcaf6b854bc2b6239fc66c68f9ba to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/20fadcaf6b854bc2b6239fc66c68f9ba 2024-11-17T22:42:38,301 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/e0b5a3e8ec8f4a4e879b8b3a6fdd8522 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/e0b5a3e8ec8f4a4e879b8b3a6fdd8522 2024-11-17T22:42:38,301 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/fb25a5b79b204a318474df26adccee6a to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/B/fb25a5b79b204a318474df26adccee6a 2024-11-17T22:42:38,303 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/0d8def3bbfd542aabe82fca2fd4736c6 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/0d8def3bbfd542aabe82fca2fd4736c6 2024-11-17T22:42:38,303 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/88ad51f3af6a43739c0ec12549dbf83e to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/88ad51f3af6a43739c0ec12549dbf83e 2024-11-17T22:42:38,304 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/d2fe44918585423fb7c9d47fa6bab75b to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/C/d2fe44918585423fb7c9d47fa6bab75b 2024-11-17T22:42:38,306 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/recovered.edits/351.seqid to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86/recovered.edits/351.seqid 2024-11-17T22:42:38,306 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/default/TestAcidGuarantees/1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,306 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T22:42:38,306 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T22:42:38,307 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T22:42:38,308 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170ae3b65cfdd74f9594f77636a53dec29_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170ae3b65cfdd74f9594f77636a53dec29_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,309 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172f3293dccf5546139c29a7b78f6cefa9_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172f3293dccf5546139c29a7b78f6cefa9_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,310 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117394cac633dab4655838cd6a828d9f515_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117394cac633dab4655838cd6a828d9f515_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,310 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117632d417b74894480be614efb6ed1b619_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117632d417b74894480be614efb6ed1b619_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,311 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117640b9e8c893f46218412870c7c84bfe2_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117640b9e8c893f46218412870c7c84bfe2_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,312 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111765307476773d4cb9b8cab0bdc012743d_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111765307476773d4cb9b8cab0bdc012743d_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,313 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ab396911b644caab1535771e7f9e6ea_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ab396911b644caab1535771e7f9e6ea_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,314 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177675699bd9174255b95acd28b87b7893_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177675699bd9174255b95acd28b87b7893_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,314 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111784a02e4c1b1444daaa3c1676c2296367_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111784a02e4c1b1444daaa3c1676c2296367_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,315 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a2131b9c066e46bfa864e2b009bfa071_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a2131b9c066e46bfa864e2b009bfa071_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,316 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117adbc6124d4a2455db4134d528715149b_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117adbc6124d4a2455db4134d528715149b_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,316 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b2736cd3c36e4e71b19e0b8acb98a1b0_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b2736cd3c36e4e71b19e0b8acb98a1b0_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,317 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b593bee5961f486c9b0341cb4ce888d2_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b593bee5961f486c9b0341cb4ce888d2_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,318 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cfceb3bea3cf4e09898251a3f6fea0e2_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cfceb3bea3cf4e09898251a3f6fea0e2_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,319 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e060b8417e774c37992331b2e6e34916_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e060b8417e774c37992331b2e6e34916_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,320 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e3568e0596a5436182586d2af525022e_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e3568e0596a5436182586d2af525022e_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,320 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e4ec0904854e4752bf7fd77108f25472_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e4ec0904854e4752bf7fd77108f25472_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,321 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eb0b22418aa14499bc5e956ee21a9103_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eb0b22418aa14499bc5e956ee21a9103_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,322 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eb1d0a321f17462985c13eb308c05118_1e5636f3ed44c24ae3f10ff764ecff86 to hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eb1d0a321f17462985c13eb308c05118_1e5636f3ed44c24ae3f10ff764ecff86 2024-11-17T22:42:38,322 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T22:42:38,324 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:38,326 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T22:42:38,328 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T22:42:38,329 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:38,329 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T22:42:38,329 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731883358329"}]},"ts":"9223372036854775807"} 2024-11-17T22:42:38,331 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T22:42:38,331 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1e5636f3ed44c24ae3f10ff764ecff86, NAME => 'TestAcidGuarantees,,1731883330244.1e5636f3ed44c24ae3f10ff764ecff86.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T22:42:38,331 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T22:42:38,331 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731883358331"}]},"ts":"9223372036854775807"} 2024-11-17T22:42:38,332 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T22:42:38,333 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T22:42:38,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-17T22:42:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-17T22:42:38,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T22:42:38,398 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T22:42:38,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 240) - Thread LEAK? -, OpenFileDescriptor=455 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 356) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4502 (was 4297) - AvailableMemoryMB LEAK? - 2024-11-17T22:42:38,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T22:42:38,405 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T22:42:38,405 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T22:42:38,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:38,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:38,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T22:42:38,405 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T22:42:38,405 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2077558802, stopped=false 2024-11-17T22:42:38,406 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1a6e40b21a48,45671,1731883188234 2024-11-17T22:42:38,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T22:42:38,407 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T22:42:38,407 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:42:38,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:42:38,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T22:42:38,408 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T22:42:38,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T22:42:38,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T22:42:38,408 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T22:42:38,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:38,408 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1a6e40b21a48,43011,1731883188895' ***** 2024-11-17T22:42:38,408 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T22:42:38,409 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T22:42:38,409 INFO [RS:0;1a6e40b21a48:43011 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T22:42:38,409 INFO [RS:0;1a6e40b21a48:43011 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T22:42:38,409 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(959): stopping server 1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:38,409 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T22:42:38,409 INFO [RS:0;1a6e40b21a48:43011 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1a6e40b21a48:43011. 2024-11-17T22:42:38,409 DEBUG [RS:0;1a6e40b21a48:43011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T22:42:38,409 DEBUG [RS:0;1a6e40b21a48:43011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:38,410 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T22:42:38,410 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T22:42:38,410 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T22:42:38,410 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T22:42:38,410 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T22:42:38,411 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T22:42:38,411 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T22:42:38,411 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T22:42:38,411 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T22:42:38,411 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T22:42:38,411 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T22:42:38,411 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T22:42:38,411 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T22:42:38,411 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-17T22:42:38,430 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/info/90696d7b14ac405d9a4304cda5eb3952 is 95, key is TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927./info:/1731883218964/DeleteFamily/seqid=0 2024-11-17T22:42:38,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742422_1598 (size=6514) 2024-11-17T22:42:38,467 INFO [regionserver/1a6e40b21a48:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T22:42:38,611 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T22:42:38,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T22:42:38,752 INFO [regionserver/1a6e40b21a48:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T22:42:38,753 INFO [regionserver/1a6e40b21a48:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T22:42:38,812 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T22:42:38,835 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/info/90696d7b14ac405d9a4304cda5eb3952 2024-11-17T22:42:38,864 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/ns/bb586c36078740c8add68cbd4a454663 is 93, key is TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927./ns:/1731883218924/DeleteFamily/seqid=0 2024-11-17T22:42:38,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742423_1599 (size=5906) 2024-11-17T22:42:39,012 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T22:42:39,212 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T22:42:39,269 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/ns/bb586c36078740c8add68cbd4a454663 2024-11-17T22:42:39,294 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/rep_barrier/fb338304f55a4d8fa02ad8553f534cc9 is 102, key is TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927./rep_barrier:/1731883218924/DeleteFamily/seqid=0 2024-11-17T22:42:39,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742424_1600 (size=6025) 2024-11-17T22:42:39,412 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T22:42:39,613 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T22:42:39,697 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/rep_barrier/fb338304f55a4d8fa02ad8553f534cc9 2024-11-17T22:42:39,716 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/table/ff3855cfd5554b8592b6294655148dc2 is 96, key is TestAcidGuarantees,,1731883192717.dde7707c5ff08c6cad672aef50110927./table:/1731883218924/DeleteFamily/seqid=0 2024-11-17T22:42:39,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742425_1601 (size=5936) 2024-11-17T22:42:39,813 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T22:42:40,013 DEBUG [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T22:42:40,120 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/table/ff3855cfd5554b8592b6294655148dc2 2024-11-17T22:42:40,123 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/info/90696d7b14ac405d9a4304cda5eb3952 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/info/90696d7b14ac405d9a4304cda5eb3952 2024-11-17T22:42:40,126 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/info/90696d7b14ac405d9a4304cda5eb3952, entries=12, sequenceid=90, filesize=6.4 K 2024-11-17T22:42:40,127 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/ns/bb586c36078740c8add68cbd4a454663 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/ns/bb586c36078740c8add68cbd4a454663 2024-11-17T22:42:40,129 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/ns/bb586c36078740c8add68cbd4a454663, entries=8, sequenceid=90, filesize=5.8 K 2024-11-17T22:42:40,130 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/rep_barrier/fb338304f55a4d8fa02ad8553f534cc9 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/rep_barrier/fb338304f55a4d8fa02ad8553f534cc9 2024-11-17T22:42:40,132 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/rep_barrier/fb338304f55a4d8fa02ad8553f534cc9, entries=6, sequenceid=90, filesize=5.9 K 2024-11-17T22:42:40,132 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/.tmp/table/ff3855cfd5554b8592b6294655148dc2 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/table/ff3855cfd5554b8592b6294655148dc2 2024-11-17T22:42:40,134 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/table/ff3855cfd5554b8592b6294655148dc2, entries=7, sequenceid=90, filesize=5.8 K 2024-11-17T22:42:40,135 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1724ms, sequenceid=90, compaction requested=false 2024-11-17T22:42:40,138 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-17T22:42:40,138 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T22:42:40,138 INFO [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T22:42:40,138 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731883358411Running coprocessor pre-close hooks at 1731883358411Disabling compacts and flushes for region at 1731883358411Disabling writes for close at 1731883358411Obtaining lock to block concurrent updates at 1731883358411Preparing flush snapshotting stores in 1588230740 at 1731883358411Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731883358412 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731883358412Flushing 1588230740/info: creating writer at 1731883358412Flushing 1588230740/info: appending metadata at 1731883358430 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731883358430Flushing 1588230740/ns: creating writer at 1731883358846 (+416 ms)Flushing 1588230740/ns: appending metadata at 1731883358864 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731883358864Flushing 1588230740/rep_barrier: creating writer at 1731883359278 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731883359294 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731883359294Flushing 1588230740/table: creating writer at 1731883359701 (+407 ms)Flushing 1588230740/table: appending metadata at 1731883359715 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731883359716 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b6162f9: reopening flushed file at 1731883360122 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71cb0c7c: reopening flushed file at 1731883360126 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@203212e3: reopening flushed file at 1731883360129 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@371747cc: reopening flushed file at 1731883360132 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1724ms, sequenceid=90, compaction requested=false at 1731883360135 (+3 ms)Writing region close event to WAL at 1731883360136 (+1 ms)Running coprocessor post-close hooks at 1731883360138 (+2 ms)Closed at 1731883360138 2024-11-17T22:42:40,138 DEBUG [RS_CLOSE_META-regionserver/1a6e40b21a48:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T22:42:40,213 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(976): stopping server 1a6e40b21a48,43011,1731883188895; all regions closed. 2024-11-17T22:42:40,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741834_1010 (size=25344) 2024-11-17T22:42:40,219 DEBUG [RS:0;1a6e40b21a48:43011 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/oldWALs 2024-11-17T22:42:40,219 INFO [RS:0;1a6e40b21a48:43011 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a6e40b21a48%2C43011%2C1731883188895.meta:.meta(num 1731883192318) 2024-11-17T22:42:40,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741833_1009 (size=13704094) 2024-11-17T22:42:40,223 DEBUG [RS:0;1a6e40b21a48:43011 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/oldWALs 2024-11-17T22:42:40,223 INFO [RS:0;1a6e40b21a48:43011 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a6e40b21a48%2C43011%2C1731883188895:(num 1731883191926) 2024-11-17T22:42:40,223 DEBUG [RS:0;1a6e40b21a48:43011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T22:42:40,223 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T22:42:40,223 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T22:42:40,223 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.ChoreService(370): Chore service for: regionserver/1a6e40b21a48:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T22:42:40,223 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T22:42:40,223 INFO [regionserver/1a6e40b21a48:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T22:42:40,224 INFO [RS:0;1a6e40b21a48:43011 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43011 2024-11-17T22:42:40,226 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a6e40b21a48,43011,1731883188895 2024-11-17T22:42:40,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T22:42:40,226 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T22:42:40,227 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a6e40b21a48,43011,1731883188895] 2024-11-17T22:42:40,228 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1a6e40b21a48,43011,1731883188895 already deleted, retry=false 2024-11-17T22:42:40,228 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1a6e40b21a48,43011,1731883188895 expired; onlineServers=0 2024-11-17T22:42:40,228 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1a6e40b21a48,45671,1731883188234' ***** 2024-11-17T22:42:40,228 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T22:42:40,228 INFO [M:0;1a6e40b21a48:45671 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T22:42:40,228 INFO [M:0;1a6e40b21a48:45671 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T22:42:40,228 DEBUG [M:0;1a6e40b21a48:45671 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T22:42:40,229 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T22:42:40,229 DEBUG [M:0;1a6e40b21a48:45671 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T22:42:40,229 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster-HFileCleaner.large.0-1731883191568 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a6e40b21a48:0:becomeActiveMaster-HFileCleaner.large.0-1731883191568,5,FailOnTimeoutGroup] 2024-11-17T22:42:40,229 DEBUG [master/1a6e40b21a48:0:becomeActiveMaster-HFileCleaner.small.0-1731883191573 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a6e40b21a48:0:becomeActiveMaster-HFileCleaner.small.0-1731883191573,5,FailOnTimeoutGroup] 2024-11-17T22:42:40,229 INFO [M:0;1a6e40b21a48:45671 {}] hbase.ChoreService(370): Chore service for: master/1a6e40b21a48:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T22:42:40,229 INFO [M:0;1a6e40b21a48:45671 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T22:42:40,229 DEBUG [M:0;1a6e40b21a48:45671 {}] master.HMaster(1795): Stopping service threads 2024-11-17T22:42:40,229 INFO [M:0;1a6e40b21a48:45671 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T22:42:40,229 INFO [M:0;1a6e40b21a48:45671 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T22:42:40,229 ERROR [M:0;1a6e40b21a48:45671 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:41421 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:41421,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-17T22:42:40,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T22:42:40,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T22:42:40,230 INFO [M:0;1a6e40b21a48:45671 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T22:42:40,230 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T22:42:40,230 DEBUG [M:0;1a6e40b21a48:45671 {}] zookeeper.ZKUtil(347): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T22:42:40,230 WARN [M:0;1a6e40b21a48:45671 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T22:42:40,231 INFO [M:0;1a6e40b21a48:45671 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/.lastflushedseqids 2024-11-17T22:42:40,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742426_1602 (size=72) 2024-11-17T22:42:40,328 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T22:42:40,328 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43011-0x1004fd798e10001, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T22:42:40,329 INFO [RS:0;1a6e40b21a48:43011 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T22:42:40,329 INFO [RS:0;1a6e40b21a48:43011 {}] regionserver.HRegionServer(1031): Exiting; stopping=1a6e40b21a48,43011,1731883188895; zookeeper connection closed. 2024-11-17T22:42:40,329 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@57fd573c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@57fd573c 2024-11-17T22:42:40,330 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-17T22:42:40,639 INFO [M:0;1a6e40b21a48:45671 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T22:42:40,639 INFO [M:0;1a6e40b21a48:45671 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T22:42:40,639 DEBUG [M:0;1a6e40b21a48:45671 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T22:42:40,639 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T22:42:40,639 DEBUG [M:0;1a6e40b21a48:45671 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T22:42:40,639 DEBUG [M:0;1a6e40b21a48:45671 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T22:42:40,639 DEBUG [M:0;1a6e40b21a48:45671 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T22:42:40,639 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=775.76 KB heapSize=954.57 KB 2024-11-17T22:42:40,659 DEBUG [M:0;1a6e40b21a48:45671 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c7252de0b4464cbd93eb9affe6bf8455 is 82, key is hbase:meta,,1/info:regioninfo/1731883192395/Put/seqid=0 2024-11-17T22:42:40,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742427_1603 (size=5672) 2024-11-17T22:42:41,063 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2212 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c7252de0b4464cbd93eb9affe6bf8455 2024-11-17T22:42:41,083 DEBUG [M:0;1a6e40b21a48:45671 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1455fa83b6bb43a39c1e6061138d9e8f is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1731883333356/Put/seqid=0 2024-11-17T22:42:41,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742428_1604 (size=44906) 2024-11-17T22:42:41,489 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775.20 KB at sequenceid=2212 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1455fa83b6bb43a39c1e6061138d9e8f 2024-11-17T22:42:41,493 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1455fa83b6bb43a39c1e6061138d9e8f 2024-11-17T22:42:41,511 DEBUG [M:0;1a6e40b21a48:45671 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4cb54add53be4f1d97d3d0714c82c159 is 69, key is 1a6e40b21a48,43011,1731883188895/rs:state/1731883191666/Put/seqid=0 2024-11-17T22:42:41,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073742429_1605 (size=5156) 2024-11-17T22:42:41,915 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2212 (bloomFilter=true), to=hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4cb54add53be4f1d97d3d0714c82c159 2024-11-17T22:42:41,920 DEBUG [M:0;1a6e40b21a48:45671 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c7252de0b4464cbd93eb9affe6bf8455 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c7252de0b4464cbd93eb9affe6bf8455 2024-11-17T22:42:41,924 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c7252de0b4464cbd93eb9affe6bf8455, entries=8, sequenceid=2212, filesize=5.5 K 2024-11-17T22:42:41,925 DEBUG [M:0;1a6e40b21a48:45671 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1455fa83b6bb43a39c1e6061138d9e8f as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1455fa83b6bb43a39c1e6061138d9e8f 2024-11-17T22:42:41,929 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1455fa83b6bb43a39c1e6061138d9e8f 2024-11-17T22:42:41,929 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1455fa83b6bb43a39c1e6061138d9e8f, entries=174, sequenceid=2212, filesize=43.9 K 2024-11-17T22:42:41,930 DEBUG [M:0;1a6e40b21a48:45671 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4cb54add53be4f1d97d3d0714c82c159 as hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4cb54add53be4f1d97d3d0714c82c159 2024-11-17T22:42:41,932 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41421/user/jenkins/test-data/6944a342-0e29-a75b-9284-c0d06230ef71/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4cb54add53be4f1d97d3d0714c82c159, entries=1, sequenceid=2212, filesize=5.0 K 2024-11-17T22:42:41,932 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.HRegion(3140): Finished flush of dataSize ~775.76 KB/794374, heapSize ~954.27 KB/977176, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1293ms, sequenceid=2212, compaction requested=false 2024-11-17T22:42:41,933 INFO [M:0;1a6e40b21a48:45671 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T22:42:41,934 DEBUG [M:0;1a6e40b21a48:45671 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731883360639Disabling compacts and flushes for region at 1731883360639Disabling writes for close at 1731883360639Obtaining lock to block concurrent updates at 1731883360639Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731883360639Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=794374, getHeapSize=977416, getOffHeapSize=0, getCellsCount=2411 at 1731883360640 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731883360640Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731883360640Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731883360659 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731883360659Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731883361066 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731883361082 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731883361082Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731883361493 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731883361511 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731883361511Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79b68c14: reopening flushed file at 1731883361919 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a9a5883: reopening flushed file at 1731883361924 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55e5bcb2: reopening flushed file at 1731883361929 (+5 ms)Finished flush of dataSize ~775.76 KB/794374, heapSize ~954.27 KB/977176, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1293ms, sequenceid=2212, compaction requested=false at 1731883361932 (+3 ms)Writing region close event to WAL at 1731883361933 (+1 ms)Closed at 1731883361933 2024-11-17T22:42:41,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39965 is added to blk_1073741830_1006 (size=939369) 2024-11-17T22:42:41,936 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T22:42:41,936 INFO [M:0;1a6e40b21a48:45671 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T22:42:41,936 INFO [M:0;1a6e40b21a48:45671 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45671 2024-11-17T22:42:41,936 INFO [M:0;1a6e40b21a48:45671 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T22:42:42,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T22:42:42,038 INFO [M:0;1a6e40b21a48:45671 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T22:42:42,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45671-0x1004fd798e10000, quorum=127.0.0.1:63234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T22:42:42,048 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T22:42:42,050 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T22:42:42,050 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T22:42:42,051 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T22:42:42,051 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/hadoop.log.dir/,STOPPED} 2024-11-17T22:42:42,054 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T22:42:42,054 WARN [BP-885076362-172.17.0.2-1731883184876 heartbeating to localhost/127.0.0.1:41421 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T22:42:42,054 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T22:42:42,054 WARN [BP-885076362-172.17.0.2-1731883184876 heartbeating to localhost/127.0.0.1:41421 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-885076362-172.17.0.2-1731883184876 (Datanode Uuid 22b3c91f-f4f7-4331-bfee-028d2ec13cf1) service to localhost/127.0.0.1:41421 2024-11-17T22:42:42,056 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/cluster_de41a222-fb27-5a57-9fde-672530362f70/data/data1/current/BP-885076362-172.17.0.2-1731883184876 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T22:42:42,056 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/cluster_de41a222-fb27-5a57-9fde-672530362f70/data/data2/current/BP-885076362-172.17.0.2-1731883184876 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T22:42:42,057 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T22:42:42,064 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T22:42:42,065 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T22:42:42,065 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T22:42:42,065 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T22:42:42,065 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d1654c7f-0889-bd2a-3621-2b9aab2d960b/hadoop.log.dir/,STOPPED} 2024-11-17T22:42:42,078 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T22:42:42,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down